/*############################# tiff2gts() ##############################*/ off_t tiff2gts(char *path, char* filename) { int fd, data_start, data_end, byte_order = 1; static off_t data_size; char *buf, dest_file_name[MAX_PATH_LENGTH], fullname[MAX_PATH_LENGTH]; struct stat stat_buf; #ifdef HAVE_SNPRINTF (void)snprintf(fullname, MAX_PATH_LENGTH, "%s/%s", path, filename); #else (void)sprintf(fullname, "%s/%s", path, filename); #endif if (stat(fullname, &stat_buf) < 0) { receive_log(ERROR_SIGN, __FILE__, __LINE__, 0L, _("Failed to stat() file `%s' : %s"), fullname, strerror(errno)); return(INCORRECT); } if (stat_buf.st_size <= (OFFSET_END + 4)) { receive_log(ERROR_SIGN, __FILE__, __LINE__, 0L, _("Could not convert file `%s'. File does not have the correct length."), filename); return(INCORRECT); } if ((buf = malloc(stat_buf.st_size)) == NULL) { receive_log(ERROR_SIGN, __FILE__, __LINE__, 0L, _("malloc() error : %s"), strerror(errno)); return(INCORRECT); } if ((fd = open(fullname, O_RDONLY, 0)) < 0) { receive_log(ERROR_SIGN, __FILE__, __LINE__, 0L, _("Failed to open() `%s' : %s"), fullname, strerror(errno)); free(buf); return(INCORRECT); } if (read(fd, buf, stat_buf.st_size) != stat_buf.st_size) { receive_log(ERROR_SIGN, __FILE__, __LINE__, 0L, _("read() error : %s"), fullname, strerror(errno)); free(buf); (void)close(fd); return(INCORRECT); } (void)close(fd); /* * Check if we have a DWD special scanner TIFF file or * some standart TIFF. The DWD special TIFF has at OFFSET_START * the offset to the start of the data, while others have * here <SOH><CR><CR><LF>. */ if ((buf[OFFSET_START] == '\001') && (buf[OFFSET_START + 1] == '\015') && (buf[OFFSET_START + 2] == '\015') && (buf[OFFSET_START + 3] == '\012')) { int ifd_offset; memcpy(&ifd_offset, &buf[4], 4); if (((buf[0] == 'I') && (buf[1] == 'I') && (*(char *)&byte_order != 1)) || ((buf[0] == 'M') && (buf[1] == 'M') && (*(char *)&byte_order == 1))) { byte_swap((char *)&ifd_offset); } data_start = OFFSET_START; data_end = ifd_offset - 1; } else { memcpy(&data_start, &buf[OFFSET_START], 4); memcpy(&data_end, &buf[OFFSET_END], 4); if (((buf[0] == 'I') && (buf[1] == 'I') && (*(char *)&byte_order != 1)) || ((buf[0] == 'M') && (buf[1] == 'M') && (*(char *)&byte_order == 1))) { byte_swap((char *)&data_start); byte_swap((char *)&data_end); } } data_size = data_end - data_start + 1; if (data_size > stat_buf.st_size) { receive_log(ERROR_SIGN, __FILE__, __LINE__, 0L, _("File %s is corrupt. Data size (%d) larger then file size (%d)."), filename, data_size, stat_buf.st_size); free(buf); return(INCORRECT); } else if (data_size <= 1) { receive_log(ERROR_SIGN, __FILE__, __LINE__, 0L, _("File %s is corrupt. Data size (%d) less then or equal to file size (%d)."), filename, data_size, stat_buf.st_size); free(buf); return(INCORRECT); } /* * Open destination file and copy data. */ #ifdef HAVE_SNPRINTF (void)snprintf(dest_file_name, MAX_PATH_LENGTH, "%s/.%s", path, filename); #else (void)sprintf(dest_file_name, "%s/.%s", path, filename); #endif if ((fd = open(dest_file_name, (O_WRONLY | O_CREAT | O_TRUNC), FILE_MODE)) < 0) { receive_log(ERROR_SIGN, __FILE__, __LINE__, 0L, _("Failed to open() %s : %s"), dest_file_name, strerror(errno)); free(buf); return(INCORRECT); } if (write(fd, &buf[data_start], data_size) != data_size) { receive_log(ERROR_SIGN, __FILE__, __LINE__, 0L, _("Failed to write() to `%s' : %s"), dest_file_name, strerror(errno)); free(buf); (void)close(fd); return(INCORRECT); } free(buf); if (close(fd) == -1) { receive_log(DEBUG_SIGN, __FILE__, __LINE__, 0L, _("close() error : %s"), strerror(errno)); } /* Remove the original file. */ if (unlink(fullname) < 0) { receive_log(WARN_SIGN, __FILE__, __LINE__, 0L, _("Failed to unlink() original TIFF file `%s' : %s"), fullname, strerror(errno)); } /* Rename file to its new name. */ #ifdef HAVE_SNPRINTF (void)snprintf(fullname, MAX_PATH_LENGTH, "%s/%s", path, filename); #else (void)sprintf(fullname, "%s/%s", path, filename); #endif if (rename(dest_file_name, fullname) < 0) { receive_log(WARN_SIGN, __FILE__, __LINE__, 0L, _("Failed to rename() file `%s' to `%s' : %s"), dest_file_name, fullname, strerror(errno)); } return(data_size); }
DESCR__S_M3 /* ** NAME ** wmoheader_from_grib - tries to create a WMO header from the content ** of a GRIB file ** ** SYNOPSIS ** void wmoheader_from_grib(char *grib_buffer, ** char *TTAAii_CCCC_YYGGgg, ** char *default_CCCC) ** ** DESCRIPTION ** The function wmoheader_from_grib() TRIES to create a WMO header ** from the buffer 'grib_buffer' of the following format: ** TTAAii_CCCC_YYGGgg ** and stores the result in the buffer 'TTAAii_CCCC_YYGGgg'. Since ** there can be so many different GRIB types this code will never ** be complete and there will always be assumptions being made ** about somne of the letters. ** ** NOTE: This code is far from complete and only works for a view ** grib types. Please do contact [email protected] if ** this does not work for your grib type, so this can be ** implemented. ** ** RETURN VALUES ** None. ** ** AUTHOR ** H.Kiehl ** ** HISTORY ** 18.08.2002 H.Kiehl Created ** */ DESCR__E_M3 #include <stdio.h> /* sprintf() */ #include <string.h> /* memcpy() */ #include "amgdefs.h" /*######################## wmoheader_from_grib() ########################*/ void wmoheader_from_grib(char *grib_buffer, char *TTAAii_CCCC_YYGGgg, char *default_CCCC) { char AA[3], CCCC[5], TT[3], ii[3], *ptr; ptr = grib_buffer + 4; if (((unsigned char)(*(ptr + 10)) < 201) || /* PDS Octet 7 */ ((unsigned char)(*(ptr + 10)) > 253)) /* PDS Octet 7 */ { /* General International Exchange */ TT[0] = 'H'; } else { /* * AWIPS * NOTE: this could also be Z, but don't know how to * determine this. :-( */ TT[0] = 'Y'; } switch ((unsigned char)(*(ptr + 12))) /* PDS Octet 9 */ { case 1 : /* Pressure */ case 2 : /* Pressure reduced to MSL */ case 3 : /* Pressure tendency (?) */ TT[1] = 'P'; break; case 6 : /* Geopotential */ case 7 : /* Geopotential height */ TT[1] = 'H'; break; case 11 : /* Temperature */ case 15 : /* Maximum temperature */ case 16 : /* Minimum temperature */ case 17 : /* Dew point temperature */ TT[1] = 'T'; break; case 33 : /* u-component of wind */ TT[1] = 'U'; break; case 34 : /* v-component of wind */ TT[1] = 'V'; break; case 39 : /* Vertical velocity (pressure) */ case 40 : /* Vertical velocity (geometric) */ TT[1] = 'O'; break; case 52 : /* Relative humidity */ TT[1] = 'R'; break; case 61 : /* Total precipitation */ TT[1] = 'E'; break; case 63 : /* Convective precipitation */ TT[1] = 'G'; break; case 71 : /* Total Cloud Cover (?) */ case 72 : /* Convective cloud cover (?) */ case 73 : /* Low cloud cover (?) */ case 74 : /* Medium cloud cover (?) */ case 75 : /* High cloud cover (?) */ TT[1] = 'B'; break; case 78 : /* Convective snow */ case 79 : /* Large scale snow */ case 99 : /* Snow melt (?) */ TT[1] = 'S'; break; case 80 : /* Water Temperature (?) */ TT[1] = 'Z'; break; case 101: /* Direction of wind waves (?) */ TT[1] = 'M'; break; case 103: /* Mean period of wind waves (?) */ TT[1] = 'D'; break; case 140: /* Categorical rain (?) */ TT[1] = 'X'; break; case 154: /* Ozone mixing ratio (?) */ TT[1] = 'Q'; break; case 187: /* Lightning (?) */ TT[1] = 'W'; break; default : /* Unknown */ receive_log(DEBUG_SIGN, __FILE__, __LINE__, 0L, _("Unknown Parameter and Unit ID %d [T2 = Z]"), (unsigned char)(*(ptr + 12))); TT[1] = 'Z'; break; } TT[2] = '\0'; switch ((unsigned char)(*(ptr + 10))) /* PDS Octet 7 */ { case 201: /* (AWIPS) */ case 21 : AA[0] = 'A'; break; case 218: /* (AWIPS) */ case 22 : AA[0] = 'B'; break; case 219: /* (AWIPS) */ case 23 : AA[0] = 'C'; break; case 220: /* (AWIPS) */ case 24 : AA[0] = 'D'; break; case 221: /* (AWIPS) */ case 25 : AA[0] = 'E'; break; case 222: /* (AWIPS) */ case 26 : AA[0] = 'F'; break; case 223: /* (AWIPS) */ case 50 : AA[0] = 'G'; break; case 202: /* (AWIPS) */ case 37 : AA[0] = 'I'; break; case 203: /* (AWIPS) */ case 38 : AA[0] = 'J'; break; case 204: /* (AWIPS) */ case 39 : AA[0] = 'K'; break; case 205: /* (AWIPS) */ case 40 : AA[0] = 'L'; break; case 206: /* (AWIPS) */ case 41 : AA[0] = 'M'; break; case 207: /* (AWIPS) */ case 42 : AA[0] = 'N'; break; case 208: /* (AWIPS) */ case 43 : AA[0] = 'O'; break; case 210: /* (AWIPS) */ case 44 : AA[0] = 'P'; break; case 214: /* (AWIPS) */ case 61 : AA[0] = 'T'; break; case 215: /* (AWIPS) */ case 62 : AA[0] = 'U'; break; case 216: /* (AWIPS) */ case 63 : AA[0] = 'V'; break; case 217: /* (AWIPS) */ case 64 : AA[0] = 'W'; break; case 255: /* non-defined grid - specified in the GDS */ if ((unsigned char)(*(ptr + 11)) & 128) /* PDS Octet 8 */ { int pds_length, gds_length, la1, lo1, la2, lo2, minus; unsigned char octet; pds_length = 0; pds_length |= (unsigned char)*(ptr + 4); pds_length <<= 8; pds_length |= (unsigned char)*(ptr + 5); pds_length <<= 8; pds_length |= (unsigned char)*(ptr + 6); gds_length = 0; gds_length |= (unsigned char)*(ptr + 4 + pds_length); gds_length <<= 8; gds_length |= (unsigned char)*(ptr + 5 + pds_length); gds_length <<= 8; gds_length |= (unsigned char)*(ptr + 6 + pds_length); /* GDS octet 11 - 13 */ octet = (unsigned char)*(ptr + pds_length + 14); if (octet & 128) { octet ^= 128; minus = YES; } else { minus = NO; } la1 = 0; la1 |= octet; la1 <<= 8; la1 |= (unsigned char)*(ptr + pds_length + 15); la1 <<= 8; la1 |= (unsigned char)*(ptr + pds_length + 16); /* GDS octet 14 - 16 */ octet = (unsigned char)*(ptr + pds_length + 17); if (octet & 128) { octet ^= 128; minus = YES; } else { minus = NO; } lo1 = 0; lo1 |= octet; lo1 <<= 8; lo1 |= (unsigned char)*(ptr + pds_length + 18); lo1 <<= 8; lo1 |= (unsigned char)*(ptr + pds_length + 19); /* GDS octet 18 - 20 */ octet = (unsigned char)*(ptr + pds_length + 21); if (octet & 128) { octet ^= 128; minus = YES; } else { minus = NO; } la2 = 0; la2 |= octet; la2 <<= 8; la2 |= (unsigned char)*(ptr + pds_length + 22); la2 <<= 8; la2 |= (unsigned char)*(ptr + pds_length + 23); if (minus == YES) { la2 = -la2; } /* GDS octet 21 - 23 */ octet = (unsigned char)*(ptr + pds_length + 24); if (octet & 128) { octet ^= 128; minus = YES; } else { minus = NO; } lo2 = 0; lo2 |= octet; lo2 <<= 8; lo2 |= (unsigned char)*(ptr + pds_length + 25); lo2 <<= 8; lo2 |= (unsigned char)*(ptr + pds_length + 26); if (minus == YES) { lo2 = -lo2; } if ((la1 >= 0) && (lo1 >= -90000) && (la2 <= 90000) && (lo2 <= 0)) { AA[0] = 'A'; } else if ((la1 >= 0) && (lo1 >= -180000) && (la2 <= 90000) && (lo2 <= -90000)) { AA[0] = 'B'; } else if ((la1 >= 0) && (lo1 >= 90000) && (la2 <= 90000) && (lo2 <= 180000)) { AA[0] = 'C'; } else if ((la1 >= 0) && (lo1 >= 0) && (la2 <= 90000) && (lo2 <= 90000)) { AA[0] = 'D'; } else if ((la1 >= -45000) && (lo1 >= -90000) && (la2 <= 45000) && (lo2 <= 0)) { AA[0] = 'E'; } else if ((la1 >= -45000) && (lo1 >= -180000) && (la2 <= 45000) && (lo2 <= -90000)) { AA[0] = 'F'; } else if ((la1 >= -45000) && (lo1 >= 90000) && (la2 <= 45000) && (lo2 <= 180000)) { AA[0] = 'G'; } else if ((la1 >= -45000) && (lo1 >= 0) && (la2 <= 45000) && (lo2 <= 90000)) { AA[0] = 'H'; } else if ((la1 >= -90000) && (lo1 >= -90000) && (la2 <= 0) && (lo2 <= 0)) { AA[0] = 'I'; } else if ((la1 >= -90000) && (lo1 >= -180000) && (la2 <= 0) && (lo2 <= -90000)) { AA[0] = 'J'; } else if ((la1 >= -90000) && (lo1 >= 90000) && (la2 <= 0) && (lo2 <= 180000)) { AA[0] = 'K'; } else if ((la1 >= -90000) && (lo1 >= 0) && (la2 <= 0) && (lo2 <= 90000)) { AA[0] = 'L'; } else if ((la1 >= 0) && (lo1 >= -45000) && (la2 <= 90000) && (lo2 <= 180000)) { AA[0] = 'T'; } else { receive_log(DEBUG_SIGN, __FILE__, __LINE__, 0L, "La1 = %d (%d.%d.%d) Lo1 = %d (%d.%d.%d) La2 = %d Lo2 = %d Scanmode = %d [A1 = X]", la1, (unsigned char)*(ptr + pds_length + 14), (unsigned char)*(ptr + pds_length + 15), (unsigned char)*(ptr + pds_length + 16), lo1, (unsigned char)*(ptr + pds_length + 17), (unsigned char)*(ptr + pds_length + 18), (unsigned char)*(ptr + pds_length + 19), la2, lo2, (unsigned char)(*(ptr + pds_length + 31))); AA[0] = 'X'; } #ifdef _WHEN_KNOWN switch ((unsigned char)(*(ptr + pds_length + 9))) /* GDS Octet 6 */ { case 0 : /* Latitude/Longitude Grid */ case 10 : /* Rotated Latitude/Longitude Grid */ case 20 : /* Stretched Latitude/Longitude Grid */ case 30 : /* Stretched and Rotated Latitude/Longitude Grid */ } receive_log(DEBUG_SIGN, __FILE__, __LINE__, 0L, "pds_length = %d gds_length = %d", pds_length, gds_length); #endif /* _WHEN_KNOWN */ } else { receive_log(DEBUG_SIGN, __FILE__, __LINE__, 0L, _("Hmmm, no GDS present %d! [A1 = X]"), (unsigned char)(*(ptr + 11))); AA[0] = 'X'; } break; case 211: if ((TT[0] == 'Y') || (TT[0] == 'Z')) { AA[0] = 'Q'; } else { receive_log(DEBUG_SIGN, __FILE__, __LINE__, 0L, _("Unknown Grid Identificator %d [A1 = X]"), (unsigned char)(*(ptr + 10))); AA[0] = 'X'; } break; case 213: if ((TT[0] == 'Y') || (TT[0] == 'Z')) { AA[0] = 'H'; } else { receive_log(DEBUG_SIGN, __FILE__, __LINE__, 0L, _("Unknown Grid Identificator %d [A1 = X]"), (unsigned char)(*(ptr + 10))); AA[0] = 'X'; } break; default : /* Unknown, lets mark this as experimental. */ receive_log(DEBUG_SIGN, __FILE__, __LINE__, 0L, _("Unknown Grid Identificator %d [A1 = X]"), (unsigned char)(*(ptr + 10))); AA[0] = 'X'; break; }
/*######################### search_old_files() ##########################*/ void search_old_files(time_t current_time) { int i, j, k, delete_file, junk_files, file_counter, queued_files, #ifdef _DELETE_LOG reason, #endif ret; time_t diff_time, pmatch_time; char *work_ptr, tmp_dir[MAX_PATH_LENGTH]; off_t queued_size_deleted, file_size; struct stat stat_buf; DIR *dp; struct dirent *p_dir; for (i = 0; i < no_of_local_dirs; i++) { if ((de[i].dir != NULL) && ((fra[de[i].fra_pos].dir_flag & DIR_DISABLED) == 0)) { (void)strcpy(tmp_dir, de[i].dir); if ((dp = opendir(tmp_dir)) == NULL) { if ((errno != ENOENT) && (errno != EACCES)) { system_log(WARN_SIGN, __FILE__, __LINE__, "Can't access directory %s : %s", tmp_dir, strerror(errno)); } } else { file_counter = 0; file_size = 0; junk_files = 0; queued_files = 0; queued_size_deleted = 0; work_ptr = tmp_dir + strlen(tmp_dir); *work_ptr++ = '/'; *work_ptr = '\0'; errno = 0; while ((p_dir = readdir(dp)) != NULL) { /* Ignore "." and "..". */ if (((p_dir->d_name[0] == '.') && (p_dir->d_name[1] == '\0')) || ((p_dir->d_name[0] == '.') && (p_dir->d_name[1] == '.') && (p_dir->d_name[2] == '\0'))) { continue; } (void)strcpy(work_ptr, p_dir->d_name); if (stat(tmp_dir, &stat_buf) < 0) { /* * Since this is a very low priority function lets not * always report when we fail to stat() a file. Maybe the * the user wants to keep some files. */ continue; } /* Sure it is a normal file? */ if (S_ISREG(stat_buf.st_mode)) { /* * Regardless of what the delete_files_flag is set, also * delete old files that are of zero length or have a * leading dot. */ diff_time = current_time - stat_buf.st_mtime; if (diff_time < 0) { diff_time = 0; } if (((p_dir->d_name[0] == '.') && (diff_time > 3600L) && ((fra[de[i].fra_pos].unknown_file_time == 0) || ((fra[de[i].fra_pos].delete_files_flag & OLD_LOCKED_FILES) && (diff_time > fra[de[i].fra_pos].locked_file_time)))) || ((diff_time > 5L) && (diff_time > fra[de[i].fra_pos].unknown_file_time))) { if ((fra[de[i].fra_pos].delete_files_flag & UNKNOWN_FILES) || (p_dir->d_name[0] == '.')) { if (p_dir->d_name[0] == '.') { if ((fra[de[i].fra_pos].delete_files_flag & OLD_LOCKED_FILES) && (diff_time > fra[de[i].fra_pos].locked_file_time)) { delete_file = YES; #ifdef _DELETE_LOG reason = DEL_OLD_LOCKED_FILE; #endif } else { delete_file = NO; } } else { if (de[i].flag & ALL_FILES) { delete_file = NO; } else { delete_file = YES; #ifdef _DELETE_LOG reason = DEL_UNKNOWN_FILE; #endif if (de[i].paused_dir == NULL) { pmatch_time = current_time; } else { pmatch_time = stat_buf.st_mtime; } for (j = 0; j < de[i].nfg; j++) { for (k = 0; ((j < de[i].nfg) && (k < de[i].fme[j].nfm)); k++) { if ((ret = pmatch(de[i].fme[j].file_mask[k], p_dir->d_name, &pmatch_time)) == 0) { delete_file = NO; j = de[i].nfg; } else if (ret == 1) { break; } } } } } if (delete_file == YES) { if (unlink(tmp_dir) == -1) { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to unlink() %s : %s", tmp_dir, strerror(errno)); } else { #ifdef _DELETE_LOG size_t dl_real_size; (void)strcpy(dl.file_name, p_dir->d_name); (void)snprintf(dl.host_name, MAX_HOSTNAME_LENGTH + 4 + 1, "%-*s %03x", MAX_HOSTNAME_LENGTH, "-", reason); *dl.file_size = stat_buf.st_size; *dl.dir_id = de[i].dir_id; *dl.job_id = 0; *dl.input_time = 0L; *dl.split_job_counter = 0; *dl.unique_number = 0; *dl.file_name_length = strlen(p_dir->d_name); dl_real_size = *dl.file_name_length + dl.size + snprintf((dl.file_name + *dl.file_name_length + 1), MAX_FILENAME_LENGTH + 1, # if SIZEOF_TIME_T == 4 "%s%c>%ld (%s %d)", # else "%s%c>%lld (%s %d)", # endif DIR_CHECK, SEPARATOR_CHAR, (pri_time_t)diff_time, __FILE__, __LINE__); if (write(dl.fd, dl.data, dl_real_size) != dl_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } #endif file_counter++; file_size += stat_buf.st_size; if ((fra[de[i].fra_pos].delete_files_flag & UNKNOWN_FILES) == 0) { junk_files++; } } } else if (fra[de[i].fra_pos].report_unknown_files == YES) { if ((fra[de[i].fra_pos].dir_flag & DIR_STOPPED) == 0) { file_counter++; file_size += stat_buf.st_size; } } } else if (fra[de[i].fra_pos].report_unknown_files == YES) { if ((fra[de[i].fra_pos].dir_flag & DIR_STOPPED) == 0) { file_counter++; file_size += stat_buf.st_size; } delete_file = NO; } else { delete_file = NO; } } else { delete_file = NO; } if ((delete_file == NO) && (p_dir->d_name[0] != '.') && (fra[de[i].fra_pos].dir_flag & DIR_STOPPED) && (fra[de[i].fra_pos].delete_files_flag & QUEUED_FILES) && (diff_time > fra[de[i].fra_pos].queued_file_time)) { if (unlink(tmp_dir) == -1) { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to unlink() %s : %s", tmp_dir, strerror(errno)); } else { #ifdef _DELETE_LOG size_t dl_real_size; (void)strcpy(dl.file_name, p_dir->d_name); (void)snprintf(dl.host_name, MAX_HOSTNAME_LENGTH + 4 + 1, "%-*s %03x", MAX_HOSTNAME_LENGTH, "-", DEL_QUEUED_FILE); *dl.file_size = stat_buf.st_size; *dl.dir_id = de[i].dir_id; *dl.job_id = 0; *dl.input_time = 0L; *dl.split_job_counter = 0; *dl.unique_number = 0; *dl.file_name_length = strlen(p_dir->d_name); dl_real_size = *dl.file_name_length + dl.size + snprintf((dl.file_name + *dl.file_name_length + 1), MAX_FILENAME_LENGTH + 1, # if SIZEOF_TIME_T == 4 "%s%c>%ld (%s %d)", # else "%s%c>%lld (%s %d)", # endif DIR_CHECK, SEPARATOR_CHAR, (pri_time_t)diff_time, __FILE__, __LINE__); if (write(dl.fd, dl.data, dl_real_size) != dl_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } #endif queued_files++; queued_size_deleted += stat_buf.st_size; } } } /* * Search queue directories for old files! */ else if ((fra[de[i].fra_pos].delete_files_flag & QUEUED_FILES) && (p_dir->d_name[0] == '.') && (S_ISDIR(stat_buf.st_mode))) { int pos; if (((pos = get_host_position(fsa, &p_dir->d_name[1], no_of_hosts)) != INCORRECT) && ((fsa[pos].host_status & DO_NOT_DELETE_DATA) == 0)) { DIR *dp_2; if ((dp_2 = opendir(tmp_dir)) == NULL) { system_log(WARN_SIGN, __FILE__, __LINE__, "Can't access directory %s : %s", tmp_dir, strerror(errno)); } else { int files_deleted = 0; off_t file_size_deleted = 0; char *work_ptr_2; work_ptr_2 = tmp_dir + strlen(tmp_dir); *work_ptr_2++ = '/'; *work_ptr_2 = '\0'; errno = 0; while ((p_dir = readdir(dp_2)) != NULL) { /* Ignore "." and "..". */ if (p_dir->d_name[0] == '.') { continue; } (void)strcpy(work_ptr_2, p_dir->d_name); if (stat(tmp_dir, &stat_buf) < 0) { /* * Since this is a very low priority function lets not * always report when we fail to stat() a file. Maybe the * the user wants to keep some files. */ continue; } /* Sure it is a normal file? */ if (S_ISREG(stat_buf.st_mode)) { diff_time = current_time - stat_buf.st_mtime; if (diff_time < 0) { diff_time = 0; } if (diff_time > fra[de[i].fra_pos].queued_file_time) { if (unlink(tmp_dir) == -1) { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to unlink() %s : %s", tmp_dir, strerror(errno)); } else { #ifdef _DELETE_LOG size_t dl_real_size; (void)strcpy(dl.file_name, p_dir->d_name); (void)snprintf(dl.host_name, MAX_HOSTNAME_LENGTH + 4 + 1, "%-*s %03x", MAX_HOSTNAME_LENGTH, fsa[pos].host_alias, DEL_QUEUED_FILE); *dl.file_size = stat_buf.st_size; *dl.dir_id = de[i].dir_id; *dl.job_id = 0; *dl.input_time = 0L; *dl.split_job_counter = 0; *dl.unique_number = 0; *dl.file_name_length = strlen(p_dir->d_name); dl_real_size = *dl.file_name_length + dl.size + snprintf((dl.file_name + *dl.file_name_length + 1), MAX_FILENAME_LENGTH + 1, # if SIZEOF_TIME_T == 4 "%s%c>%ld (%s %d)", # else "%s%c>%lld (%s %d)", # endif DIR_CHECK, SEPARATOR_CHAR, (pri_time_t)diff_time, __FILE__, __LINE__); if (write(dl.fd, dl.data, dl_real_size) != dl_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } #endif files_deleted++; file_size_deleted += stat_buf.st_size; } } } errno = 0; } /* while ((p_dir = readdir(dp_2)) != NULL) */ if (errno) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not readdir() %s : %s", tmp_dir, strerror(errno)); } if (files_deleted > 0) { queued_files += files_deleted; queued_size_deleted += file_size_deleted; ABS_REDUCE_QUEUE(de[i].fra_pos, files_deleted, file_size_deleted); } /* Don't forget to close the directory. */ if (closedir(dp_2) < 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not close directory %s : %s", tmp_dir, strerror(errno)); } } } } errno = 0; } /* * NOTE: The ENOENT is when it catches a file that is just * being renamed (lock DOT). */ if ((errno) && (errno != ENOENT)) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not readdir() %s : %s", tmp_dir, strerror(errno)); } /* Don't forget to close the directory. */ if (closedir(dp) < 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not close directory %s : %s", tmp_dir, strerror(errno)); } /* Remove file name from directory name. */ *(work_ptr - 1) = '\0'; /* Tell user there are old files in this directory. */ if (((file_counter - junk_files) > 0) && (fra[de[i].fra_pos].report_unknown_files == YES) && ((fra[de[i].fra_pos].delete_files_flag & UNKNOWN_FILES) == 0)) { p_fra = &fra[de[i].fra_pos]; if (file_size >= GIGABYTE) { receive_log(WARN_SIGN, NULL, 0, current_time, "There are %d (%d GiB) old (>%dh) files in %s. @%x", file_counter - junk_files, (int)(file_size / 1073741824), fra[de[i].fra_pos].unknown_file_time / 3600, tmp_dir, de[i].dir_id); } else if (file_size >= MEGABYTE) { receive_log(WARN_SIGN, NULL, 0, current_time, "There are %d (%d MiB) old (>%dh) files in %s. @%x", file_counter - junk_files, (int)(file_size / 1048576), fra[de[i].fra_pos].unknown_file_time / 3600, tmp_dir, de[i].dir_id); } else if (file_size >= KILOBYTE) { receive_log(WARN_SIGN, NULL, 0, current_time, "There are %d (%d KiB) old (>%dh) files in %s. @%x", file_counter - junk_files, (int)(file_size / 1024), fra[de[i].fra_pos].unknown_file_time / 3600, tmp_dir, de[i].dir_id); } else { receive_log(WARN_SIGN, NULL, 0, current_time, #if SIZEOF_OFF_T == 4 "There are %d (%ld bytes) old (>%dh) files in %s. @%x", #else "There are %d (%lld bytes) old (>%dh) files in %s. @%x", #endif file_counter - junk_files, (pri_off_t)file_size, fra[de[i].fra_pos].unknown_file_time / 3600, tmp_dir, de[i].dir_id); } } if (junk_files > 0) { p_fra = &fra[de[i].fra_pos]; receive_log(DEBUG_SIGN, NULL, 0, current_time, "Deleted %d file(s) (>%dh) that where of length 0 or had a leading dot, in %s. @%x", junk_files, fra[de[i].fra_pos].unknown_file_time / 3600, tmp_dir, de[i].dir_id); } if (queued_files > 0) { p_fra = &fra[de[i].fra_pos]; if (queued_size_deleted >= GIGABYTE) { receive_log(DEBUG_SIGN, NULL, 0, current_time, "Deleted %d (%d GiB) queued file(s), in %s. @%x", queued_files, (int)(queued_size_deleted / 1073741824), tmp_dir, de[i].dir_id); } else if (queued_size_deleted >= MEGABYTE) { receive_log(DEBUG_SIGN, NULL, 0, current_time, "Deleted %d (%d MiB) queued file(s), in %s. @%x", queued_files, (int)(queued_size_deleted / 1048576), tmp_dir, de[i].dir_id); } else if (queued_size_deleted >= KILOBYTE) { receive_log(DEBUG_SIGN, NULL, 0, current_time, "Deleted %d (%d KiB) queued file(s), in %s. @%x", queued_files, (int)(queued_size_deleted / 1024), tmp_dir, de[i].dir_id); } else { receive_log(DEBUG_SIGN, NULL, 0, current_time, #if SIZEOF_OFF_T == 4 "Deleted %d (%ld bytes) queued file(s), in %s. @%x", #else "Deleted %d (%lld bytes) queued file(s), in %s. @%x", #endif queued_files, (pri_off_t)queued_size_deleted, tmp_dir, de[i].dir_id); } } } } } return; }
DESCR__S_M3 /* ** NAME ** wmo2ascii - removes SOH, ETX and the two carriage returns from ** the given file ** ** SYNOPSIS ** int wmo2ascii(char *filename, off_t *filesize) ** ** DESCRIPTION ** The function wmo2ascii will remove SOH, ETX and the two carriage ** returns from the given WMO file. It will convert the following ** text from: ** ** <SOH><CR><CR><LF>nnn<CR><CR><LF> ** WMO header<CR><CR><LF>WMO message<CR><CR><LF><ETX> ** ** to: ** ** WMO header<LF>WMO message<LF> ** ** in addition <CR><CR><LF> will be changed to <LF>. ** The file name will not change. ** ** RETURN VALUES ** On failure INCORRECT is returned otherwise SUCCESS will be ** returned. ** ** AUTHOR ** H.Kiehl ** ** HISTORY ** 16.07.2002 H.Kiehl Created ** */ DESCR__E_M3 #include <stdio.h> /* rename() */ #include <string.h> /* strerror() */ #include <stdlib.h> /* malloc(), free() */ #include <ctype.h> /* isdigit() */ #include <unistd.h> /* read(), write(), close(), unlink() */ #include <sys/types.h> #include <sys/stat.h> /* fstat() */ #include <fcntl.h> #include <errno.h> #include "amgdefs.h" /*############################# wmo2ascii() #############################*/ int wmo2ascii(char *file_path, char *p_file_name, off_t *filesize) { int from_fd, ret = INCORRECT; char from[MAX_PATH_LENGTH]; *filesize = 0; #ifdef HAVE_SNPRINTF (void)snprintf(from, MAX_PATH_LENGTH, "%s/%s", file_path, p_file_name); #else (void)sprintf(from, "%s/%s", file_path, p_file_name); #endif if ((from_fd = open(from, O_RDONLY)) == -1) { receive_log(WARN_SIGN, __FILE__, __LINE__, 0L, _("wmo2ascii(): Failed to open() `%s' : %s"), from, strerror(errno)); } else { struct stat stat_buf; if (fstat(from_fd, &stat_buf) == -1) { receive_log(WARN_SIGN, __FILE__, __LINE__, 0L, _("wmo2ascii(): Failed to stat() `%s' : %s"), from, strerror(errno)); } else { if (stat_buf.st_size > 0) { int to_fd; char to[MAX_PATH_LENGTH]; #ifdef HAVE_SNPRINTF (void)snprintf(to, MAX_PATH_LENGTH, "%s/.wmo2ascii", file_path); #else (void)sprintf(to, "%s/.wmo2ascii", file_path); #endif if ((to_fd = open(to, O_WRONLY | O_CREAT | O_TRUNC, stat_buf.st_mode)) == -1) { receive_log(WARN_SIGN, __FILE__, __LINE__, 0L, _("wmo2ascii(): Failed to open() `%s' : %s"), to, strerror(errno)); } else { char *read_buffer, *write_buffer; if (((read_buffer = malloc(stat_buf.st_blksize + 1)) == NULL) || ((write_buffer = malloc(stat_buf.st_blksize + 1)) == NULL)) { receive_log(ERROR_SIGN, __FILE__, __LINE__, 0L, _("wmo2ascii(): malloc() error : %s"), strerror(errno)); free(read_buffer); } else { ssize_t bytes_buffered; if ((bytes_buffered = read(from_fd, read_buffer, stat_buf.st_blksize)) == -1) { receive_log(WARN_SIGN, __FILE__, __LINE__, 0L, _("wmo2ascii(): Failed to read() `%s' : %s"), from, strerror(errno)); } else { off_t length, length_done; char *read_ptr = read_buffer, *ptr_end = read_buffer + bytes_buffered, *write_ptr = write_buffer; /* * Check if there is a message counter, if yes * remove it as well. */ if ((bytes_buffered > 10) && (*read_ptr == '\001') && (*(read_ptr + 1) == '\015') && (*(read_ptr + 2) == '\015') && (*(read_ptr + 3) == '\012')) { read_ptr += 4; if ((isdigit((int)(*read_ptr))) && (isdigit((int)(*(read_ptr + 1)))) && (isdigit((int)(*(read_ptr + 2))))) { char *tmp_ptr = read_ptr + 3; while (tmp_ptr < ptr_end) { if (isdigit((int)(*tmp_ptr)) == 0) { break; } tmp_ptr++; } if ((*tmp_ptr == '\015') && (*(tmp_ptr + 1) == '\015') && (*(tmp_ptr + 2) == '\012')) { read_ptr = tmp_ptr + 3; } } } while (read_ptr < ptr_end) { if ((*read_ptr != '\001') && (*read_ptr != '\003') && (*read_ptr != '\015')) { *write_ptr = *read_ptr; write_ptr++; } read_ptr++; } length = write_ptr - write_buffer; if (writen(to_fd, write_buffer, length, stat_buf.st_blksize) != length) { receive_log(WARN_SIGN, __FILE__, __LINE__, 0L, _("wmo2ascii(): Failed to writen() `%s' : %s"), to, strerror(errno)); } else { length_done = length; while (bytes_buffered == stat_buf.st_blksize) { if ((bytes_buffered = read(from_fd, read_buffer, stat_buf.st_blksize)) == -1) { receive_log(WARN_SIGN, __FILE__, __LINE__, 0L, _("wmo2ascii(): Failed to read() `%s' : %s"), from, strerror(errno)); (void)close(from_fd); (void)close(to_fd); (void)unlink(to); free(read_buffer); free(write_buffer); return(ret); } read_ptr = read_buffer, ptr_end = read_buffer + bytes_buffered, write_ptr = write_buffer; while (read_ptr < ptr_end) { if ((*read_ptr != '\001') && (*read_ptr != '\003') && (*read_ptr != '\015')) { *write_ptr = *read_ptr; write_ptr++; } read_ptr++; } length = write_ptr - write_buffer; if (writen(to_fd, write_buffer, length, stat_buf.st_blksize) != length) { receive_log(WARN_SIGN, __FILE__, __LINE__, 0L, _("wmo2ascii(): Failed to writen() `%s' : %s"), to, strerror(errno)); (void)close(from_fd); (void)close(to_fd); (void)unlink(to); free(read_buffer); free(write_buffer); return(ret); } length_done += length; } if (unlink(from) == -1) { receive_log(WARN_SIGN, __FILE__, __LINE__, 0L, _("wmo2ascii(): Failed to unlink() `%s' : %s"), from, strerror(errno)); } if (rename(to, from) == -1) { receive_log(WARN_SIGN, __FILE__, __LINE__, 0L, _("wmo2ascii(): Failed to rename() `%s' to `%s' : %s"), to, from, strerror(errno)); } else { ret = SUCCESS; *filesize = length_done; } } } free(read_buffer); free(write_buffer); } if (close(to_fd) == -1) { receive_log(DEBUG_SIGN, __FILE__, __LINE__, 0L, _("wmo2ascii(): Failed to close() `%s' : %s"), to, strerror(errno)); } } } } if (close(from_fd) == -1) { receive_log(DEBUG_SIGN, __FILE__, __LINE__, 0L, _("wmo2ascii(): Failed to close() `%s' : %s"), from, strerror(errno)); } } return(ret); }
/*$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ main() $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$*/ int main(int argc, char *argv[]) { #ifdef _WITH_BURST_2 int cb2_ret; #endif int current_toggle, exit_status = TRANSFER_SUCCESS, j, fd, status, loops, rest, blocksize, *wmo_counter, wmo_counter_fd = -1; #ifdef WITH_ARCHIVE_COPY_INFO unsigned int archived_copied = 0; #endif off_t no_of_bytes; time_t connected, #ifdef _WITH_BURST_2 diff_time, #endif end_transfer_time_file, start_transfer_time_file = 0, last_update_time, now; #ifdef _OUTPUT_LOG clock_t end_time = 0, start_time = 0; struct tms tmsdummy; #endif char *p_file_name_buffer, *buffer, fullname[MAX_PATH_LENGTH + 1], file_path[MAX_PATH_LENGTH]; clock_t clktck; struct stat stat_buf; struct job *p_db; #ifdef SA_FULLDUMP struct sigaction sact; #endif CHECK_FOR_VERSION(argc, argv); #ifdef SA_FULLDUMP /* * When dumping core sure we do a FULL core dump! */ sact.sa_handler = SIG_DFL; sact.sa_flags = SA_FULLDUMP; sigemptyset(&sact.sa_mask); if (sigaction(SIGSEGV, &sact, NULL) == -1) { system_log(ERROR_SIGN, __FILE__, __LINE__, "sigaction() error : %s", strerror(errno)); exit(INCORRECT); } #endif /* Do some cleanups when we exit. */ if (atexit(sf_wmo_exit) != 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not register exit function : %s", strerror(errno)); exit(INCORRECT); } /* Initialise variables. */ local_file_counter = 0; files_to_send = init_sf(argc, argv, file_path, WMO_FLAG); p_db = &db; if ((clktck = sysconf(_SC_CLK_TCK)) <= 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not get clock ticks per second : %s", strerror(errno)); exit(INCORRECT); } if (fsa->trl_per_process > 0) { if (fsa->trl_per_process < fsa->block_size) { blocksize = fsa->trl_per_process; } else { blocksize = fsa->block_size; } } else { blocksize = fsa->block_size; } if ((signal(SIGINT, sig_kill) == SIG_ERR) || (signal(SIGQUIT, sig_exit) == SIG_ERR) || (signal(SIGTERM, SIG_IGN) == SIG_ERR) || (signal(SIGSEGV, sig_segv) == SIG_ERR) || (signal(SIGBUS, sig_bus) == SIG_ERR) || (signal(SIGHUP, SIG_IGN) == SIG_ERR) || (signal(SIGPIPE, SIG_IGN) == SIG_ERR)) { system_log(ERROR_SIGN, __FILE__, __LINE__, "signal() error : %s", strerror(errno)); exit(INCORRECT); } /* Now determine the real hostname. */ if (db.toggle_host == YES) { if (fsa->host_toggle == HOST_ONE) { (void)strcpy(db.hostname, fsa->real_hostname[HOST_TWO - 1]); current_toggle = HOST_TWO; } else { (void)strcpy(db.hostname, fsa->real_hostname[HOST_ONE - 1]); current_toggle = HOST_ONE; } } else { (void)strcpy(db.hostname, fsa->real_hostname[(int)(fsa->host_toggle - 1)]); current_toggle = (int)fsa->host_toggle; } /* Connect to remote WMO-server. */ #ifdef FTP_CTRL_KEEP_ALIVE_INTERVAL if (fsa->protocol_options & AFD_TCP_KEEPALIVE) { timeout_flag = transfer_timeout - 5; if (timeout_flag < MIN_KEEP_ALIVE_INTERVAL) { timeout_flag = MIN_KEEP_ALIVE_INTERVAL; } } #else timeout_flag = OFF; #endif if ((status = wmo_connect(db.hostname, db.port, db.sndbuf_size)) != SUCCESS) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "WMO connection to <%s> at port %d failed (%d).", db.hostname, db.port, status); exit(eval_timeout(CONNECT_ERROR)); } else { if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Connected to port %d.", db.port); } } connected = time(NULL); /* Inform FSA that we have finished connecting and */ /* will now start to transfer data. */ if (gsf_check_fsa(p_db) != NEITHER) { #ifdef LOCK_DEBUG lock_region_w(fsa_fd, db.lock_offset + LOCK_CON, __FILE__, __LINE__); #else lock_region_w(fsa_fd, db.lock_offset + LOCK_CON); #endif fsa->job_status[(int)db.job_no].connect_status = WMO_ACTIVE; fsa->job_status[(int)db.job_no].no_of_files = files_to_send; fsa->connections += 1; #ifdef LOCK_DEBUG unlock_region(fsa_fd, db.lock_offset + LOCK_CON, __FILE__, __LINE__); #else unlock_region(fsa_fd, db.lock_offset + LOCK_CON); #endif } /* Allocate buffer to read data from the source file. */ if ((buffer = malloc(blocksize + 1 + 4 /* For bulletin end. */)) == NULL) { system_log(ERROR_SIGN, __FILE__, __LINE__, "malloc() error : %s", strerror(errno)); exit(ALLOC_ERROR); } if (db.special_flag & WITH_SEQUENCE_NUMBER) { char counter_file_name[MAX_FILENAME_LENGTH]; (void)snprintf(counter_file_name, MAX_FILENAME_LENGTH, "/%s.%d", db.host_alias, db.port); if ((wmo_counter_fd = open_counter_file(counter_file_name, &wmo_counter)) < 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to open counter file `%s'.", counter_file_name); } } #ifdef _WITH_BURST_2 do { if (burst_2_counter > 0) { if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "WMO Bursting."); } } #endif /* Send all files. */ p_file_name_buffer = file_name_buffer; p_file_size_buffer = file_size_buffer; last_update_time = time(NULL); local_file_size = 0; for (files_send = 0; files_send < files_to_send; files_send++) { (void)snprintf(fullname, MAX_PATH_LENGTH + 1, "%s/%s", file_path, p_file_name_buffer); if (*p_file_size_buffer > 0) { int end_length = 0, header_length = 0, length_type_indicator = 10; if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_size_in_use = *p_file_size_buffer; (void)strcpy(fsa->job_status[(int)db.job_no].file_name_in_use, p_file_name_buffer); } /* Open local file. */ #ifdef O_LARGEFILE if ((fd = open(fullname, O_RDONLY | O_LARGEFILE)) == -1) #else if ((fd = open(fullname, O_RDONLY)) == -1) #endif { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to open local file `%s' : %s", fullname, strerror(errno)); wmo_quit(); exit(OPEN_LOCAL_ERROR); } if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Open local file `%s'", fullname); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { start_time = times(&tmsdummy); } #endif /* * When the contents does not contain a bulletin header * it must be stored in the file name. */ if (db.special_flag & FILE_NAME_IS_HEADER) { int space_count; char *ptr = p_file_name_buffer; buffer[length_type_indicator] = 1; /* SOH */ buffer[length_type_indicator + 1] = '\015'; /* CR */ buffer[length_type_indicator + 2] = '\015'; /* CR */ buffer[length_type_indicator + 3] = '\012'; /* LF */ header_length = 4; space_count = 0; if (wmo_counter_fd > 0) { if (next_counter(wmo_counter_fd, wmo_counter, MAX_WMO_COUNTER) < 0) { close_counter_file(wmo_counter_fd, &wmo_counter); wmo_counter_fd = -1; wmo_counter = NULL; system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to get next WMO counter."); } else { if (*wmo_counter < 10) { buffer[length_type_indicator + header_length] = '0'; buffer[length_type_indicator + header_length + 1] = '0'; buffer[length_type_indicator + header_length + 2] = *wmo_counter + '0'; } else if (*wmo_counter < 100) { buffer[length_type_indicator + header_length] = '0'; buffer[length_type_indicator + header_length + 1] = (*wmo_counter / 10) + '0'; buffer[length_type_indicator + header_length + 2] = (*wmo_counter % 10) + '0'; } else if (*wmo_counter < 1000) { buffer[length_type_indicator + header_length] = ((*wmo_counter / 100) % 10) + '0'; buffer[length_type_indicator + header_length + 1] = ((*wmo_counter / 10) % 10) + '0'; buffer[length_type_indicator + header_length + 2] = (*wmo_counter % 10) + '0'; } buffer[length_type_indicator + header_length + 3] = '\015'; /* CR */ buffer[length_type_indicator + header_length + 4] = '\015'; /* CR */ buffer[length_type_indicator + header_length + 5] = '\012'; /* LF */ header_length += 6; } } /* if (wmo_counter_fd > 0) */ for (;;) { while ((*ptr != '_') && (*ptr != '-') && (*ptr != ' ') && (*ptr != '\0') && (*ptr != '.') && (*ptr != ';')) { buffer[length_type_indicator + header_length] = *ptr; header_length++; ptr++; } if ((*ptr == '\0') || (*ptr == '.') || (*ptr == ';')) { break; } else { if (space_count == 2) { if ((isalpha((int)(*(ptr + 1)))) && (isalpha((int)(*(ptr + 2)))) && (isalpha((int)(*(ptr + 3))))) { buffer[length_type_indicator + header_length] = ' '; buffer[length_type_indicator + header_length + 1] = *(ptr + 1); buffer[length_type_indicator + header_length + 2] = *(ptr + 2); buffer[length_type_indicator + header_length + 3] = *(ptr + 3); header_length += 4; } break; } else { buffer[length_type_indicator + header_length] = ' '; header_length++; ptr++; space_count++; } } } /* for (;;) */ buffer[length_type_indicator + header_length] = '\015'; /* CR */ buffer[length_type_indicator + header_length + 1] = '\015'; /* CR */ buffer[length_type_indicator + header_length + 2] = '\012'; /* LF */ header_length += 3; end_length = 4; } /* Read (local) and write (remote) file. */ no_of_bytes = 0; loops = (length_type_indicator + header_length + *p_file_size_buffer) / blocksize; rest = (length_type_indicator + header_length + *p_file_size_buffer) % blocksize; if ((db.special_flag & FILE_NAME_IS_HEADER) && (rest == 0)) { loops--; rest = blocksize; } /* Write length and type indicator. */ (void)snprintf(buffer, 9, "%08lu", (unsigned long)(*p_file_size_buffer + header_length + end_length)); if (db.transfer_mode == 'I') { buffer[length_type_indicator - 2] = 'B'; buffer[length_type_indicator - 1] = 'I'; } else if (db.transfer_mode == 'A') { buffer[length_type_indicator - 2] = 'A'; buffer[length_type_indicator - 1] = 'N'; } else { buffer[length_type_indicator - 2] = 'F'; buffer[length_type_indicator - 1] = 'X'; } if (fsa->trl_per_process > 0) { init_limit_transfer_rate(); } if (fsa->protocol_options & TIMEOUT_TRANSFER) { start_transfer_time_file = time(NULL); } for (;;) { for (j = 0; j < loops; j++) { #ifdef _SIMULATE_SLOW_TRANSFER (void)sleep(_SIMULATE_SLOW_TRANSFER); #endif if ((status = read(fd, (buffer + length_type_indicator + header_length), (blocksize - length_type_indicator - header_length))) != (blocksize - length_type_indicator - header_length)) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Could not read() local file `%s' : %s", fullname, strerror(errno)); wmo_quit(); exit(READ_LOCAL_ERROR); } if ((status = wmo_write(buffer, blocksize)) != SUCCESS) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to write block from file `%s' to remote port %d [%d].", p_file_name_buffer, db.port, status); wmo_quit(); exit(eval_timeout(WRITE_REMOTE_ERROR)); } if (fsa->trl_per_process > 0) { limit_transfer_rate(blocksize, fsa->trl_per_process, clktck); } no_of_bytes += blocksize; if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_size_in_use_done = no_of_bytes; fsa->job_status[(int)db.job_no].file_size_done += blocksize; fsa->job_status[(int)db.job_no].bytes_send += blocksize; if (fsa->protocol_options & TIMEOUT_TRANSFER) { end_transfer_time_file = time(NULL); if (end_transfer_time_file < start_transfer_time_file) { start_transfer_time_file = end_transfer_time_file; } else { if ((end_transfer_time_file - start_transfer_time_file) > transfer_timeout) { trans_log(INFO_SIGN, __FILE__, __LINE__, NULL, NULL, #if SIZEOF_TIME_T == 4 "Transfer timeout reached for `%s' after %ld seconds.", #else "Transfer timeout reached for `%s' after %lld seconds.", #endif fsa->job_status[(int)db.job_no].file_name_in_use, (pri_time_t)(end_transfer_time_file - start_transfer_time_file)); wmo_quit(); exitflag = 0; exit(STILL_FILES_TO_SEND); } } } } if (length_type_indicator > 0) { length_type_indicator = 0; header_length = 0; } } /* for (j = 0; j < loops; j++) */ if (rest > 0) { if ((status = read(fd, (buffer + length_type_indicator + header_length), (rest - length_type_indicator - header_length))) != (rest - length_type_indicator - header_length)) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Could not read() local file `%s' : %s", fullname, strerror(errno)); wmo_quit(); exit(READ_LOCAL_ERROR); } if (end_length == 4) { buffer[rest] = '\015'; buffer[rest + 1] = '\015'; buffer[rest + 2] = '\012'; buffer[rest + 3] = 3; /* ETX */ } if ((status = wmo_write(buffer, rest + end_length)) != SUCCESS) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to write rest of file to remote port %d [%d].", p_file_name_buffer, db.port, status); wmo_quit(); exit(eval_timeout(WRITE_REMOTE_ERROR)); } if (fsa->trl_per_process > 0) { limit_transfer_rate(rest + end_length, fsa->trl_per_process, clktck); } no_of_bytes += rest + end_length; if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_size_in_use_done = no_of_bytes; fsa->job_status[(int)db.job_no].file_size_done += rest; fsa->job_status[(int)db.job_no].bytes_send += rest; } } /* * Since there are always some users sending files to the * AFD not in dot notation, lets check here if this is really * the EOF. * If not lets continue so long until we hopefully have reached * the EOF. * NOTE: This is NOT a fool proof way. There must be a better * way! */ if (fstat(fd, &stat_buf) == -1) { (void)rec(transfer_log_fd, DEBUG_SIGN, "Hmmm. Failed to stat() `%s' : %s (%s %d)\n", fullname, strerror(errno), __FILE__, __LINE__); break; } else { if (stat_buf.st_size > *p_file_size_buffer) { char sign[LOG_SIGN_LENGTH]; if (db.special_flag & SILENT_NOT_LOCKED_FILE) { (void)memcpy(sign, DEBUG_SIGN, LOG_SIGN_LENGTH); } else { (void)memcpy(sign, WARN_SIGN, LOG_SIGN_LENGTH); } loops = (stat_buf.st_size - *p_file_size_buffer) / blocksize; rest = (stat_buf.st_size - *p_file_size_buffer) % blocksize; *p_file_size_buffer = stat_buf.st_size; /* * Give a warning in the receive log, so some action * can be taken against the originator. */ receive_log(sign, __FILE__, __LINE__, 0L, db.id.job, "File `%s' for host %s was DEFINITELY send without any locking. #%x", p_file_name_buffer, fsa->host_dsp_name, db.id.job); } else { break; } } } /* for (;;) */ if (db.special_flag & WMO_CHECK_ACKNOWLEDGE) { int ret; if ((ret = wmo_check_reply()) == INCORRECT) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Failed to receive reply from port %d for file %s.", db.port, p_file_name_buffer); wmo_quit(); exit(eval_timeout(CHECK_REPLY_ERROR)); } else if (ret == NEGATIV_ACKNOWLEDGE) { trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL, "Received negative acknowledge from remote port %d for file %s.", db.port, p_file_name_buffer); } } #ifdef _OUTPUT_LOG if (db.output_log == YES) { end_time = times(&tmsdummy); } #endif /* Close local file. */ if (close(fd) == -1) { (void)rec(transfer_log_fd, WARN_SIGN, "%-*s[%d]: Failed to close() local file %s : %s (%s %d)\n", MAX_HOSTNAME_LENGTH, tr_hostname, (int)db.job_no, p_file_name_buffer, strerror(errno), __FILE__, __LINE__); /* * Since we usually do not send more then 100 files and * sf_wmo() will exit(), there is no point in stopping * the transmission. */ } } else { trans_log(INFO_SIGN, __FILE__, __LINE__, NULL, NULL, "File `%s' is of zero length, ignoring.", p_file_name_buffer); } /* Update FSA, one file transmitted. */ if (gsf_check_fsa(p_db) != NEITHER) { fsa->job_status[(int)db.job_no].file_name_in_use[0] = '\0'; fsa->job_status[(int)db.job_no].no_of_files_done++; fsa->job_status[(int)db.job_no].file_size_in_use = 0; fsa->job_status[(int)db.job_no].file_size_in_use_done = 0; local_file_size += *p_file_size_buffer; local_file_counter += 1; now = time(NULL); if (now >= (last_update_time + LOCK_INTERVAL_TIME)) { last_update_time = now; update_tfc(local_file_counter, local_file_size, p_file_size_buffer, files_to_send, files_send, now); local_file_size = 0; local_file_counter = 0; } } #ifdef _WITH_TRANS_EXEC if (db.special_flag & TRANS_EXEC) { trans_exec(file_path, fullname, p_file_name_buffer, clktck); } #endif #ifdef _OUTPUT_LOG if (db.output_log == YES) { if (ol_fd == -2) { # ifdef WITHOUT_FIFO_RW_SUPPORT output_log_fd(&ol_fd, &ol_readfd, &db.output_log); # else output_log_fd(&ol_fd, &db.output_log); # endif } if ((ol_fd > -1) && (ol_data == NULL)) { output_log_ptrs(&ol_retries, &ol_job_number, &ol_data, /* Pointer to buffer. */ &ol_file_name, &ol_file_name_length, &ol_archive_name_length, &ol_file_size, &ol_unl, &ol_size, &ol_transfer_time, &ol_output_type, db.host_alias, (current_toggle - 1), WMO, &db.output_log); } } #endif /* Now archive file if necessary. */ if ((db.archive_time > 0) && (p_db->archive_dir[0] != FAILED_TO_CREATE_ARCHIVE_DIR)) { #ifdef WITH_ARCHIVE_COPY_INFO int ret; #endif /* * By telling the function archive_file() that this * is the first time to archive a file for this job * (in struct p_db) it does not always have to check * whether the directory has been created or not. And * we ensure that we do not create duplicate names * when adding db.archive_time to msg_name. */ #ifdef WITH_ARCHIVE_COPY_INFO if ((ret = archive_file(file_path, p_file_name_buffer, p_db)) < 0) #else if (archive_file(file_path, p_file_name_buffer, p_db) < 0) #endif { if (fsa->debug > NORMAL_MODE) { trans_db_log(ERROR_SIGN, __FILE__, __LINE__, NULL, "Failed to archive file `%s'", p_file_name_buffer); } /* * NOTE: We _MUST_ delete the file we just send, * else the file directory will run full! */ if (unlink(fullname) == -1) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not unlink() local file `%s' after sending it successfully : %s", fullname, strerror(errno)); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { (void)memcpy(ol_file_name, db.p_unique_name, db.unl); (void)strcpy(ol_file_name + db.unl, p_file_name_buffer); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; *ol_file_size = *p_file_size_buffer; *ol_job_number = fsa->job_status[(int)db.job_no].job_id; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = end_time - start_time; *ol_archive_name_length = 0; *ol_output_type = OT_NORMAL_DELIVERED + '0'; ol_real_size = *ol_file_name_length + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } } #endif } else { if (fsa->debug > NORMAL_MODE) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Archived file `%s'", p_file_name_buffer); } #ifdef WITH_ARCHIVE_COPY_INFO if (ret == DATA_COPIED) { archived_copied++; } #endif #ifdef _OUTPUT_LOG if (db.output_log == YES) { (void)memcpy(ol_file_name, db.p_unique_name, db.unl); (void)strcpy(ol_file_name + db.unl, p_file_name_buffer); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; (void)strcpy(&ol_file_name[*ol_file_name_length + 1], &db.archive_dir[db.archive_offset]); *ol_file_size = *p_file_size_buffer; *ol_job_number = fsa->job_status[(int)db.job_no].job_id; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = end_time - start_time; *ol_archive_name_length = (unsigned short)strlen(&ol_file_name[*ol_file_name_length + 1]); *ol_output_type = OT_NORMAL_DELIVERED + '0'; ol_real_size = *ol_file_name_length + *ol_archive_name_length + 1 + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } } #endif } } else { #ifdef WITH_UNLINK_DELAY int unlink_loops = 0; try_again_unlink: #endif /* Delete the file we just have send. */ if (unlink(fullname) == -1) { #ifdef WITH_UNLINK_DELAY if ((errno == EBUSY) && (unlink_loops < 20)) { (void)my_usleep(100000L); unlink_loops++; goto try_again_unlink; } #endif system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not unlink() local file %s after sending it successfully : %s", fullname, strerror(errno)); } #ifdef _OUTPUT_LOG if (db.output_log == YES) { (void)memcpy(ol_file_name, db.p_unique_name, db.unl); (void)strcpy(ol_file_name + db.unl, p_file_name_buffer); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; *ol_file_size = *p_file_size_buffer; *ol_job_number = fsa->job_status[(int)db.job_no].job_id; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = end_time - start_time; *ol_archive_name_length = 0; *ol_output_type = OT_NORMAL_DELIVERED + '0'; ol_real_size = *ol_file_name_length + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s", strerror(errno)); } } #endif } /* * After each successful transfer set error * counter to zero, so that other jobs can be * started. */ if (gsf_check_fsa(p_db) != NEITHER) { if ((*p_file_size_buffer > 0) && (fsa->error_counter > 0)) { int fd, #ifdef WITHOUT_FIFO_RW_SUPPORT readfd, #endif j; char fd_wake_up_fifo[MAX_PATH_LENGTH]; #ifdef LOCK_DEBUG lock_region_w(fsa_fd, db.lock_offset + LOCK_EC, __FILE__, __LINE__); #else lock_region_w(fsa_fd, db.lock_offset + LOCK_EC); #endif fsa->error_counter = 0; /* * Wake up FD! */ (void)snprintf(fd_wake_up_fifo, MAX_PATH_LENGTH, "%s%s%s", p_work_dir, FIFO_DIR, FD_WAKE_UP_FIFO); #ifdef WITHOUT_FIFO_RW_SUPPORT if (open_fifo_rw(fd_wake_up_fifo, &readfd, &fd) == -1) #else if ((fd = open(fd_wake_up_fifo, O_RDWR)) == -1) #endif { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to open() FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } else { if (write(fd, "", 1) != 1) { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to write() to FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } #ifdef WITHOUT_FIFO_RW_SUPPORT if (close(readfd) == -1) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Failed to close() FIFO %s (read) : %s", fd_wake_up_fifo, strerror(errno)); } #endif if (close(fd) == -1) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Failed to close() FIFO %s : %s", fd_wake_up_fifo, strerror(errno)); } } /* * Remove the error condition (NOT_WORKING) from all jobs * of this host. */ for (j = 0; j < fsa->allowed_transfers; j++) { if ((j != db.job_no) && (fsa->job_status[j].connect_status == NOT_WORKING)) { fsa->job_status[j].connect_status = DISCONNECT; } } fsa->error_history[0] = 0; fsa->error_history[1] = 0; #ifdef LOCK_DEBUG unlock_region(fsa_fd, db.lock_offset + LOCK_EC, __FILE__, __LINE__); #else unlock_region(fsa_fd, db.lock_offset + LOCK_EC); #endif #ifdef LOCK_DEBUG lock_region_w(fsa_fd, db.lock_offset + LOCK_HS, __FILE__, __LINE__); #else lock_region_w(fsa_fd, db.lock_offset + LOCK_HS); #endif now = time(NULL); if (now > fsa->end_event_handle) { fsa->host_status &= ~(EVENT_STATUS_FLAGS | AUTO_PAUSE_QUEUE_STAT); if (fsa->end_event_handle > 0L) { fsa->end_event_handle = 0L; } if (fsa->start_event_handle > 0L) { fsa->start_event_handle = 0L; } } else { fsa->host_status &= ~(EVENT_STATUS_STATIC_FLAGS | AUTO_PAUSE_QUEUE_STAT); } #ifdef LOCK_DEBUG unlock_region(fsa_fd, db.lock_offset + LOCK_HS, __FILE__, __LINE__); #else unlock_region(fsa_fd, db.lock_offset + LOCK_HS); #endif /* * Since we have successfully transmitted a file, no need to * have the queue stopped anymore. */ if (fsa->host_status & AUTO_PAUSE_QUEUE_STAT) { char sign[LOG_SIGN_LENGTH]; error_action(fsa->host_alias, "stop", HOST_ERROR_ACTION); event_log(0L, EC_HOST, ET_EXT, EA_ERROR_END, "%s", fsa->host_alias); if ((fsa->host_status & HOST_ERROR_OFFLINE_STATIC) || (fsa->host_status & HOST_ERROR_OFFLINE) || (fsa->host_status & HOST_ERROR_OFFLINE_T)) { (void)memcpy(sign, OFFLINE_SIGN, LOG_SIGN_LENGTH); } else { (void)memcpy(sign, INFO_SIGN, LOG_SIGN_LENGTH); } trans_log(sign, __FILE__, __LINE__, NULL, NULL, "Starting input queue that was stopped by init_afd."); event_log(0L, EC_HOST, ET_AUTO, EA_START_QUEUE, "%s", fsa->host_alias); } } /* if (fsa->error_counter > 0) */ #ifdef WITH_ERROR_QUEUE if (fsa->host_status & ERROR_QUEUE_SET) { remove_from_error_queue(db.id.job, fsa, db.fsa_pos, fsa_fd); } #endif if (fsa->host_status & HOST_ACTION_SUCCESS) { error_action(fsa->host_alias, "start", HOST_SUCCESS_ACTION); } } p_file_name_buffer += MAX_FILENAME_LENGTH; p_file_size_buffer++; } /* for (files_send = 0; files_send < files_to_send; files_send++) */ #ifdef WITH_ARCHIVE_COPY_INFO if (archived_copied > 0) { trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL, "Copied %u files to archive.", archived_copied); archived_copied = 0; } #endif if (local_file_counter) { if (gsf_check_fsa(p_db) != NEITHER) { update_tfc(local_file_counter, local_file_size, p_file_size_buffer, files_to_send, files_send, time(NULL)); local_file_size = 0; local_file_counter = 0; } } /* * Remove file directory, but only when all files have * been transmitted. */ if ((files_to_send == files_send) || (files_to_send < 1)) { if (rmdir(file_path) < 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to remove directory %s : %s", file_path, strerror(errno)); } } else { system_log(WARN_SIGN, __FILE__, __LINE__, "There are still %d files for %s. Will NOT remove this job!", files_to_send - files_send, file_path); exit_status = STILL_FILES_TO_SEND; } #ifdef _WITH_BURST_2 burst_2_counter++; diff_time = time(NULL) - connected; if (((fsa->protocol_options & KEEP_CONNECTED_DISCONNECT) && (db.keep_connected > 0) && (diff_time > db.keep_connected)) || ((db.disconnect > 0) && (diff_time > db.disconnect))) { cb2_ret = NO; break; } } while ((cb2_ret = check_burst_sf(file_path, &files_to_send, 0, # ifdef _WITH_INTERRUPT_JOB 0, # endif # ifdef _OUTPUT_LOG &ol_fd, # endif # ifndef AFDBENCH_CONFIG NULL, # endif NULL)) == YES); burst_2_counter--; if (cb2_ret == NEITHER) { exit_status = STILL_FILES_TO_SEND; } #endif /* _WITH_BURST_2 */ free(buffer); /* Disconnect from remote port. */ wmo_quit(); if ((fsa != NULL) && (fsa->debug > NORMAL_MODE)) { trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL, "Disconnected from port %d.", db.port); } if (wmo_counter_fd > 0) { close_counter_file(wmo_counter_fd, &wmo_counter); } exitflag = 0; exit(exit_status); }