/*############################ convert_jid() ############################*/
char *
convert_jid(int old_jid_fd,
char *old_job_id_data_file,
size_t *old_jid_size,
int old_no_of_job_ids,
unsigned char old_version,
unsigned char new_version)
{
char *ptr;
struct stat stat_buf;
if ((old_version == 1) && (new_version == 2))
{
int i;
size_t new_size;
struct job_id_data_1 *old_jid;
struct job_id_data_2 *new_jid;
/* Get the size of the old JID file. */
if (fstat(old_jid_fd, &stat_buf) < 0)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"Failed to fstat() %s : %s",
old_job_id_data_file, strerror(errno));
*old_jid_size = -1;
return(NULL);
}
else
{
if (stat_buf.st_size > 0)
{
#ifdef HAVE_MMAP
if ((ptr = mmap(NULL, stat_buf.st_size, (PROT_READ | PROT_WRITE),
MAP_SHARED, old_jid_fd, 0)) == (caddr_t) -1)
#else
if ((ptr = mmap_emu(NULL, stat_buf.st_size, (PROT_READ | PROT_WRITE),
MAP_SHARED, old_job_id_data_file, 0)) == (caddr_t) -1)
#endif
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"Failed to mmap() to %s : %s",
old_job_id_data_file, strerror(errno));
*old_jid_size = -1;
return(NULL);
}
}
else
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"FSA file %s is empty.", old_job_id_data_file);
*old_jid_size = -1;
return(NULL);
}
}
ptr += AFD_WORD_OFFSET_1;
old_jid = (struct job_id_data_1 *)ptr;
new_size = ((old_no_of_job_ids / JOB_ID_DATA_STEP_SIZE) + 1) *
JOB_ID_DATA_STEP_SIZE * sizeof(struct job_id_data_2);
if ((ptr = malloc(new_size)) == NULL)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"malloc() error [%d %d] : %s",
old_no_of_job_ids, new_size, strerror(errno));
ptr = (char *)old_jid;
ptr -= AFD_WORD_OFFSET_1;
#ifdef HAVE_MMAP
if (munmap(ptr, stat_buf.st_size) == -1)
#else
if (munmap_emu(ptr) == -1)
#endif
{
system_log(WARN_SIGN, __FILE__, __LINE__,
"Failed to munmap() %s : %s",
old_job_id_data_file, strerror(errno));
}
*old_jid_size = -1;
return(NULL);
}
(void)memset(ptr, 0, new_size);
new_jid = (struct job_id_data_2 *)ptr;
/*
* Copy all the old data into the new region.
*/
for (i = 0; i < old_no_of_job_ids; i++)
{
(void)strcpy(new_jid[i].host_alias, old_jid[i].host_alias);
(void)strcpy(new_jid[i].recipient, old_jid[i].recipient);
(void)memcpy(new_jid[i].soptions, old_jid[i].soptions, MAX_OPTION_LENGTH_1);
(void)memcpy(new_jid[i].loptions, old_jid[i].loptions, MAX_OPTION_LENGTH_1);
new_jid[i].no_of_soptions = old_jid[i].no_of_soptions;
new_jid[i].no_of_loptions = old_jid[i].no_of_loptions;
new_jid[i].priority = old_jid[i].priority;
new_jid[i].dir_id_pos = old_jid[i].dir_id_pos;
new_jid[i].dir_config_id = old_jid[i].dir_config_id;
new_jid[i].file_mask_id = old_jid[i].file_mask_id;
new_jid[i].dir_id = old_jid[i].dir_id;
new_jid[i].job_id = old_jid[i].job_id;
new_jid[i].host_id = get_str_checksum(new_jid[i].host_alias);
new_jid[i].recipient_id = get_str_checksum(new_jid[i].recipient);
}
ptr = (char *)old_jid;
ptr -= AFD_WORD_OFFSET_1;
/*
* Resize the old JID to the size of new one and then copy
* the new structure into it.
*/
if ((ptr = mmap_resize(old_jid_fd, ptr, new_size + AFD_WORD_OFFSET_2)) == (caddr_t) -1)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"Failed to mmap_resize() %s : %s",
old_job_id_data_file, strerror(errno));
free((void *)new_jid);
return(NULL);
}
ptr += AFD_WORD_OFFSET_2;
(void)memcpy(ptr, new_jid, new_size);
free((void *)new_jid);
ptr -= AFD_WORD_OFFSET_2;
*(int *)ptr = old_no_of_job_ids;
*old_jid_size = new_size + AFD_WORD_OFFSET_2;
system_log(INFO_SIGN, NULL, 0,
"Converted JID from verion %d to %d.",
(int)old_version, (int)new_version);
system_log(DEBUG_SIGN, __FILE__, __LINE__,
#if SIZEOF_OFF_T == 4
# if SIZEOF_SIZE_T == 4
"filesize old: %ld (%d) new: %ld (%d) Number of jobs: %d",
# else
"filesize old: %ld (%d) new: %lld (%d) Number of jobs: %d",
# endif
#else
# if SIZEOF_SIZE_T == 4
"filesize old: %lld (%d) new: %ld (%d) Number of jobs: %d",
# else
"filesize old: %lld (%d) new: %lld (%d) Number of jobs: %d",
# endif
#endif
(pri_off_t)stat_buf.st_size, (int)sizeof(struct job_id_data_1),
(pri_size_t)new_size + AFD_WORD_OFFSET_2,
(int)sizeof(struct job_id_data_2), old_no_of_job_ids);
}
else if ((old_version == 1) && (new_version == 3))
{
int i;
size_t new_size;
struct job_id_data_1 *old_jid;
struct job_id_data_3 *new_jid;
/* Get the size of the old JID file. */
if (fstat(old_jid_fd, &stat_buf) < 0)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"Failed to fstat() %s : %s",
old_job_id_data_file, strerror(errno));
*old_jid_size = -1;
return(NULL);
}
else
{
if (stat_buf.st_size > 0)
{
#ifdef HAVE_MMAP
if ((ptr = mmap(NULL, stat_buf.st_size, (PROT_READ | PROT_WRITE),
MAP_SHARED, old_jid_fd, 0)) == (caddr_t) -1)
#else
if ((ptr = mmap_emu(NULL, stat_buf.st_size, (PROT_READ | PROT_WRITE),
MAP_SHARED, old_job_id_data_file, 0)) == (caddr_t) -1)
#endif
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"Failed to mmap() to %s : %s",
old_job_id_data_file, strerror(errno));
*old_jid_size = -1;
return(NULL);
}
}
else
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"FSA file %s is empty.", old_job_id_data_file);
*old_jid_size = -1;
return(NULL);
}
}
ptr += AFD_WORD_OFFSET_1;
old_jid = (struct job_id_data_1 *)ptr;
new_size = ((old_no_of_job_ids / JOB_ID_DATA_STEP_SIZE) + 1) *
JOB_ID_DATA_STEP_SIZE * sizeof(struct job_id_data_3);
if ((ptr = malloc(new_size)) == NULL)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"malloc() error [%d %d] : %s",
old_no_of_job_ids, new_size, strerror(errno));
ptr = (char *)old_jid;
ptr -= AFD_WORD_OFFSET_1;
#ifdef HAVE_MMAP
if (munmap(ptr, stat_buf.st_size) == -1)
#else
if (munmap_emu(ptr) == -1)
#endif
{
system_log(WARN_SIGN, __FILE__, __LINE__,
"Failed to munmap() %s : %s",
old_job_id_data_file, strerror(errno));
}
*old_jid_size = -1;
return(NULL);
}
(void)memset(ptr, 0, new_size);
new_jid = (struct job_id_data_3 *)ptr;
/*
* Copy all the old data into the new region.
*/
for (i = 0; i < old_no_of_job_ids; i++)
{
(void)strcpy(new_jid[i].host_alias, old_jid[i].host_alias);
(void)strcpy(new_jid[i].recipient, old_jid[i].recipient);
(void)memcpy(new_jid[i].soptions, old_jid[i].soptions, MAX_OPTION_LENGTH_1);
(void)memcpy(new_jid[i].loptions, old_jid[i].loptions, MAX_OPTION_LENGTH_1);
new_jid[i].creation_time = 0L;
new_jid[i].special_flag = 0;
new_jid[i].no_of_soptions = old_jid[i].no_of_soptions;
new_jid[i].no_of_loptions = old_jid[i].no_of_loptions;
new_jid[i].priority = old_jid[i].priority;
new_jid[i].dir_id_pos = old_jid[i].dir_id_pos;
new_jid[i].dir_config_id = old_jid[i].dir_config_id;
new_jid[i].file_mask_id = old_jid[i].file_mask_id;
new_jid[i].dir_id = old_jid[i].dir_id;
new_jid[i].job_id = old_jid[i].job_id;
new_jid[i].host_id = get_str_checksum(new_jid[i].host_alias);
new_jid[i].recipient_id = get_str_checksum(new_jid[i].recipient);
}
ptr = (char *)old_jid;
ptr -= AFD_WORD_OFFSET_1;
/*
* Resize the old JID to the size of new one and then copy
* the new structure into it.
*/
if ((ptr = mmap_resize(old_jid_fd, ptr, new_size + AFD_WORD_OFFSET_3)) == (caddr_t) -1)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"Failed to mmap_resize() %s : %s",
old_job_id_data_file, strerror(errno));
free((void *)new_jid);
return(NULL);
}
ptr += AFD_WORD_OFFSET_3;
(void)memcpy(ptr, new_jid, new_size);
free((void *)new_jid);
ptr -= AFD_WORD_OFFSET_3;
*(int *)ptr = old_no_of_job_ids;
*old_jid_size = new_size + AFD_WORD_OFFSET_3;
system_log(INFO_SIGN, NULL, 0,
"Converted JID from verion %d to %d.",
(int)old_version, (int)new_version);
system_log(DEBUG_SIGN, __FILE__, __LINE__,
#if SIZEOF_OFF_T == 4
# if SIZEOF_SIZE_T == 4
"filesize old: %ld (%d) new: %ld (%d) Number of jobs: %d",
# else
"filesize old: %ld (%d) new: %lld (%d) Number of jobs: %d",
# endif
#else
# if SIZEOF_SIZE_T == 4
"filesize old: %lld (%d) new: %ld (%d) Number of jobs: %d",
# else
"filesize old: %lld (%d) new: %lld (%d) Number of jobs: %d",
# endif
#endif
(pri_off_t)stat_buf.st_size, (int)sizeof(struct job_id_data_1),
(pri_size_t)new_size + AFD_WORD_OFFSET_3,
(int)sizeof(struct job_id_data_3), old_no_of_job_ids);
}
else if ((old_version == 2) && (new_version == 3))
{
int i;
size_t new_size;
struct job_id_data_2 *old_jid;
struct job_id_data_3 *new_jid;
/* Get the size of the old JID file. */
if (fstat(old_jid_fd, &stat_buf) < 0)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"Failed to fstat() %s : %s",
old_job_id_data_file, strerror(errno));
*old_jid_size = -1;
return(NULL);
}
else
{
if (stat_buf.st_size > 0)
{
#ifdef HAVE_MMAP
if ((ptr = mmap(NULL, stat_buf.st_size, (PROT_READ | PROT_WRITE),
MAP_SHARED, old_jid_fd, 0)) == (caddr_t) -1)
#else
if ((ptr = mmap_emu(NULL, stat_buf.st_size, (PROT_READ | PROT_WRITE),
MAP_SHARED, old_job_id_data_file, 0)) == (caddr_t) -1)
#endif
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"Failed to mmap() to %s : %s",
old_job_id_data_file, strerror(errno));
*old_jid_size = -1;
return(NULL);
}
}
else
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"FSA file %s is empty.", old_job_id_data_file);
*old_jid_size = -1;
return(NULL);
}
}
ptr += AFD_WORD_OFFSET_2;
old_jid = (struct job_id_data_2 *)ptr;
new_size = ((old_no_of_job_ids / JOB_ID_DATA_STEP_SIZE) + 1) *
JOB_ID_DATA_STEP_SIZE * sizeof(struct job_id_data_3);
if ((ptr = malloc(new_size)) == NULL)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"malloc() error [%d %d] : %s",
old_no_of_job_ids, new_size, strerror(errno));
ptr = (char *)old_jid;
ptr -= AFD_WORD_OFFSET_2;
#ifdef HAVE_MMAP
if (munmap(ptr, stat_buf.st_size) == -1)
#else
if (munmap_emu(ptr) == -1)
#endif
{
system_log(WARN_SIGN, __FILE__, __LINE__,
"Failed to munmap() %s : %s",
old_job_id_data_file, strerror(errno));
}
*old_jid_size = -1;
return(NULL);
}
(void)memset(ptr, 0, new_size);
new_jid = (struct job_id_data_3 *)ptr;
/*
* Copy all the old data into the new region.
*/
for (i = 0; i < old_no_of_job_ids; i++)
{
(void)strcpy(new_jid[i].host_alias, old_jid[i].host_alias);
(void)strcpy(new_jid[i].recipient, old_jid[i].recipient);
(void)memcpy(new_jid[i].soptions, old_jid[i].soptions, MAX_OPTION_LENGTH_2);
(void)memcpy(new_jid[i].loptions, old_jid[i].loptions, MAX_OPTION_LENGTH_2);
new_jid[i].creation_time = 0L;
new_jid[i].special_flag = 0;
new_jid[i].no_of_soptions = old_jid[i].no_of_soptions;
new_jid[i].no_of_loptions = old_jid[i].no_of_loptions;
new_jid[i].priority = old_jid[i].priority;
new_jid[i].dir_id_pos = old_jid[i].dir_id_pos;
new_jid[i].dir_config_id = old_jid[i].dir_config_id;
new_jid[i].file_mask_id = old_jid[i].file_mask_id;
new_jid[i].dir_id = old_jid[i].dir_id;
new_jid[i].job_id = old_jid[i].job_id;
new_jid[i].host_id = old_jid[i].host_id;
new_jid[i].recipient_id = old_jid[i].recipient_id;
}
ptr = (char *)old_jid;
ptr -= AFD_WORD_OFFSET_2;
/*
* Resize the old JID to the size of new one and then copy
* the new structure into it.
*/
if ((ptr = mmap_resize(old_jid_fd, ptr, new_size + AFD_WORD_OFFSET_3)) == (caddr_t) -1)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"Failed to mmap_resize() %s : %s",
old_job_id_data_file, strerror(errno));
free((void *)new_jid);
return(NULL);
}
ptr += AFD_WORD_OFFSET_3;
(void)memcpy(ptr, new_jid, new_size);
free((void *)new_jid);
ptr -= AFD_WORD_OFFSET_3;
*(int *)ptr = old_no_of_job_ids;
*old_jid_size = new_size + AFD_WORD_OFFSET_3;
system_log(INFO_SIGN, NULL, 0,
"Converted JID from verion %d to %d.",
(int)old_version, (int)new_version);
system_log(DEBUG_SIGN, __FILE__, __LINE__,
#if SIZEOF_OFF_T == 4
# if SIZEOF_SIZE_T == 4
"filesize old: %ld (%d) new: %ld (%d) Number of jobs: %d",
# else
"filesize old: %ld (%d) new: %lld (%d) Number of jobs: %d",
# endif
#else
# if SIZEOF_SIZE_T == 4
"filesize old: %lld (%d) new: %ld (%d) Number of jobs: %d",
# else
"filesize old: %lld (%d) new: %lld (%d) Number of jobs: %d",
# endif
#endif
(pri_off_t)stat_buf.st_size, (int)sizeof(struct job_id_data_2),
(pri_size_t)new_size + AFD_WORD_OFFSET_3,
(int)sizeof(struct job_id_data_3), old_no_of_job_ids);
}
else
{
system_log(ERROR_SIGN, NULL, 0,
"Don't know how to convert a version %d JID to version %d.",
old_version, new_version);
ptr = NULL;
}
return(ptr);
}
/*+++++++++++++++++++++++++++++ check_list() ++++++++++++++++++++++++++++*/
static int
check_list(char *file,
off_t file_size,
time_t file_mtime,
int *files_to_retrieve,
off_t *file_size_to_retrieve,
int *more_files_in_list)
{
int i;
if ((fra[db.fra_pos].stupid_mode == YES) ||
(fra[db.fra_pos].remove == YES))
{
for (i = 0; i < no_of_listed_files; i++)
{
if (CHECK_STRCMP(rl[i].file_name, file) == 0)
{
rl[i].in_list = YES;
if (((rl[i].assigned == 0) || (rl[i].retrieved == YES)) &&
(((db.special_flag & OLD_ERROR_JOB) == 0) ||
#ifdef LOCK_DEBUG
(lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i),
__FILE__, __LINE__) == LOCK_IS_NOT_SET)
#else
(lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i)) == LOCK_IS_NOT_SET)
#endif
))
{
int ret;
rl[i].file_mtime = file_mtime;
rl[i].got_date = YES;
rl[i].size = file_size;
rl[i].prev_size = 0;
if ((fra[db.fra_pos].ignore_size == -1) ||
((fra[db.fra_pos].gt_lt_sign & ISIZE_EQUAL) &&
(fra[db.fra_pos].ignore_size == rl[i].size)) ||
((fra[db.fra_pos].gt_lt_sign & ISIZE_LESS_THEN) &&
(fra[db.fra_pos].ignore_size < rl[i].size)) ||
((fra[db.fra_pos].gt_lt_sign & ISIZE_GREATER_THEN) &&
(fra[db.fra_pos].ignore_size > rl[i].size)))
{
if (fra[db.fra_pos].ignore_file_time == 0)
{
*file_size_to_retrieve += rl[i].size;
*files_to_retrieve += 1;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
if ((fra[db.fra_pos].stupid_mode == YES) ||
(fra[db.fra_pos].remove == YES) ||
((*files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
(*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size)))
#else
if ((*files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
(*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size))
#endif
{
rl[i].retrieved = NO;
rl[i].assigned = (unsigned char)db.job_no + 1;
}
else
{
rl[i].assigned = 0;
*file_size_to_retrieve -= rl[i].size;
*files_to_retrieve -= 1;
*more_files_in_list = YES;
}
ret = 0;
}
else
{
time_t diff_time;
diff_time = current_time - rl[i].file_mtime;
if (((fra[db.fra_pos].gt_lt_sign & IFTIME_EQUAL) &&
(fra[db.fra_pos].ignore_file_time == diff_time)) ||
((fra[db.fra_pos].gt_lt_sign & IFTIME_LESS_THEN) &&
(fra[db.fra_pos].ignore_file_time < diff_time)) ||
((fra[db.fra_pos].gt_lt_sign & IFTIME_GREATER_THEN) &&
(fra[db.fra_pos].ignore_file_time > diff_time)))
{
*file_size_to_retrieve += rl[i].size;
*files_to_retrieve += 1;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
if ((fra[db.fra_pos].stupid_mode == YES) ||
(fra[db.fra_pos].remove == YES) ||
((*files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
(*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size)))
#else
if ((*files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
(*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size))
#endif
{
rl[i].retrieved = NO;
rl[i].assigned = (unsigned char)db.job_no + 1;
}
else
{
rl[i].assigned = 0;
*file_size_to_retrieve -= rl[i].size;
*files_to_retrieve -= 1;
*more_files_in_list = YES;
}
ret = 0;
}
else
{
ret = 1;
}
}
#ifdef DEBUG_ASSIGNMENT
trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL,
# if SIZEOF_OFF_T == 4
"%s assigned %d: file_name=%s assigned=%d size=%ld",
# else
"%s assigned %d: file_name=%s assigned=%d size=%lld",
# endif
(fra[db.fra_pos].ls_data_alias[0] == '\0') ? fra[db.fra_pos].dir_alias : fra[db.fra_pos].ls_data_alias,
i, rl[i].file_name, (int)rl[i].assigned,
(pri_off_t)rl[i].size);
#endif /* DEBUG_ASSIGNMENT */
}
else
{
ret = 1;
}
if (db.special_flag & OLD_ERROR_JOB)
{
#ifdef LOCK_DEBUG
unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i),
__FILE__, __LINE__);
#else
unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i));
#endif
}
return(ret);
}
else
{
return(1);
}
}
} /* for (i = 0; i < no_of_listed_files; i++) */
}
else
{
/* Check if this file is in the list. */
for (i = 0; i < no_of_listed_files; i++)
{
if (CHECK_STRCMP(rl[i].file_name, file) == 0)
{
rl[i].in_list = YES;
if ((rl[i].assigned != 0) ||
((fra[db.fra_pos].stupid_mode == GET_ONCE_ONLY) &&
(rl[i].retrieved == YES)))
{
return(1);
}
if (((db.special_flag & OLD_ERROR_JOB) == 0) ||
#ifdef LOCK_DEBUG
(lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i),
__FILE__, __LINE__) == LOCK_IS_NOT_SET)
#else
(lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i)) == LOCK_IS_NOT_SET)
#endif
)
{
int ret;
off_t prev_size = 0;
if (rl[i].file_mtime != file_mtime)
{
rl[i].file_mtime = file_mtime;
rl[i].retrieved = NO;
rl[i].assigned = 0;
}
rl[i].got_date = YES;
if (rl[i].size != file_size)
{
prev_size = rl[i].size;
rl[i].size = file_size;
rl[i].retrieved = NO;
rl[i].assigned = 0;
}
if (rl[i].retrieved == NO)
{
if ((fra[db.fra_pos].ignore_size == -1) ||
((fra[db.fra_pos].gt_lt_sign & ISIZE_EQUAL) &&
(fra[db.fra_pos].ignore_size == rl[i].size)) ||
((fra[db.fra_pos].gt_lt_sign & ISIZE_LESS_THEN) &&
(fra[db.fra_pos].ignore_size < rl[i].size)) ||
((fra[db.fra_pos].gt_lt_sign & ISIZE_GREATER_THEN) &&
(fra[db.fra_pos].ignore_size > rl[i].size)))
{
off_t size_to_retrieve;
if ((rl[i].got_date == NO) ||
(fra[db.fra_pos].ignore_file_time == 0))
{
if ((fra[db.fra_pos].stupid_mode == APPEND_ONLY) &&
(rl[i].size > prev_size))
{
size_to_retrieve = rl[i].size - prev_size;
}
else
{
size_to_retrieve = rl[i].size;
}
rl[i].prev_size = prev_size;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
if ((fra[db.fra_pos].stupid_mode == YES) ||
(fra[db.fra_pos].remove == YES) ||
(((*files_to_retrieve + 1) < fra[db.fra_pos].max_copied_files) &&
((*file_size_to_retrieve + size_to_retrieve) < fra[db.fra_pos].max_copied_file_size)))
#else
if (((*files_to_retrieve + 1) < fra[db.fra_pos].max_copied_files) &&
((*file_size_to_retrieve + size_to_retrieve) < fra[db.fra_pos].max_copied_file_size))
#endif
{
rl[i].assigned = (unsigned char)db.job_no + 1;
*file_size_to_retrieve += size_to_retrieve;
*files_to_retrieve += 1;
}
else
{
rl[i].assigned = 0;
*more_files_in_list = YES;
}
ret = 0;
}
else
{
time_t diff_time;
diff_time = current_time - rl[i].file_mtime;
if (((fra[db.fra_pos].gt_lt_sign & IFTIME_EQUAL) &&
(fra[db.fra_pos].ignore_file_time == diff_time)) ||
((fra[db.fra_pos].gt_lt_sign & IFTIME_LESS_THEN) &&
(fra[db.fra_pos].ignore_file_time < diff_time)) ||
((fra[db.fra_pos].gt_lt_sign & IFTIME_GREATER_THEN) &&
(fra[db.fra_pos].ignore_file_time > diff_time)))
{
if ((fra[db.fra_pos].stupid_mode == APPEND_ONLY) &&
(rl[i].size > prev_size))
{
size_to_retrieve = rl[i].size - prev_size;
}
else
{
size_to_retrieve = rl[i].size;
}
rl[i].prev_size = prev_size;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
if ((fra[db.fra_pos].stupid_mode == YES) ||
(fra[db.fra_pos].remove == YES) ||
(((*files_to_retrieve + 1) < fra[db.fra_pos].max_copied_files) &&
((*file_size_to_retrieve + size_to_retrieve) < fra[db.fra_pos].max_copied_file_size)))
#else
if (((*files_to_retrieve + 1) < fra[db.fra_pos].max_copied_files) &&
((*file_size_to_retrieve + size_to_retrieve) < fra[db.fra_pos].max_copied_file_size))
#endif
{
rl[i].assigned = (unsigned char)db.job_no + 1;
*file_size_to_retrieve += size_to_retrieve;
*files_to_retrieve += 1;
}
else
{
rl[i].assigned = 0;
*more_files_in_list = YES;
}
ret = 0;
}
else
{
ret = 1;
}
}
#ifdef DEBUG_ASSIGNMENT
trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL,
# if SIZEOF_OFF_T == 4
"%s assigned %d: file_name=%s assigned=%d size=%ld",
# else
"%s assigned %d: file_name=%s assigned=%d size=%lld",
# endif
(fra[db.fra_pos].ls_data_alias[0] == '\0') ? fra[db.fra_pos].dir_alias : fra[db.fra_pos].ls_data_alias,
i, rl[i].file_name, (int)rl[i].assigned,
(pri_off_t)rl[i].size);
#endif /* DEBUG_ASSIGNMENT */
}
else
{
ret = 1;
}
}
else
{
ret = 1;
}
if (db.special_flag & OLD_ERROR_JOB)
{
#ifdef LOCK_DEBUG
unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i),
__FILE__, __LINE__);
#else
unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i));
#endif
}
return(ret);
}
else
{
return(1);
}
}
} /* for (i = 0; i < no_of_listed_files; i++) */
}
/* Add this file to the list. */
if ((no_of_listed_files != 0) &&
((no_of_listed_files % RETRIEVE_LIST_STEP_SIZE) == 0))
{
char *ptr;
size_t new_size = (((no_of_listed_files / RETRIEVE_LIST_STEP_SIZE) + 1) *
RETRIEVE_LIST_STEP_SIZE * sizeof(struct retrieve_list)) +
AFD_WORD_OFFSET;
ptr = (char *)rl - AFD_WORD_OFFSET;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
if ((fra[db.fra_pos].stupid_mode == YES) ||
(fra[db.fra_pos].remove == YES))
{
if ((ptr = realloc(ptr, new_size)) == NULL)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"realloc() error : %s", strerror(errno));
(void)ftp_quit();
exit(INCORRECT);
}
}
else
{
#endif
if ((ptr = mmap_resize(rl_fd, ptr, new_size)) == (caddr_t) -1)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"mmap_resize() error : %s", strerror(errno));
(void)ftp_quit();
exit(INCORRECT);
}
rl_size = new_size;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
}
#endif
if (no_of_listed_files < 0)
{
system_log(DEBUG_SIGN, __FILE__, __LINE__,
"Hmmm, no_of_listed_files = %d", no_of_listed_files);
no_of_listed_files = 0;
}
*(int *)ptr = no_of_listed_files;
ptr += AFD_WORD_OFFSET;
rl = (struct retrieve_list *)ptr;
}
(void)strcpy(rl[no_of_listed_files].file_name, file);
rl[no_of_listed_files].retrieved = NO;
rl[no_of_listed_files].in_list = YES;
rl[no_of_listed_files].size = file_size;
rl[no_of_listed_files].prev_size = 0;
rl[no_of_listed_files].file_mtime = file_mtime;
rl[no_of_listed_files].got_date = YES;
if ((fra[db.fra_pos].ignore_size == -1) ||
((fra[db.fra_pos].gt_lt_sign & ISIZE_EQUAL) &&
(fra[db.fra_pos].ignore_size == rl[no_of_listed_files].size)) ||
((fra[db.fra_pos].gt_lt_sign & ISIZE_LESS_THEN) &&
(fra[db.fra_pos].ignore_size < rl[no_of_listed_files].size)) ||
((fra[db.fra_pos].gt_lt_sign & ISIZE_GREATER_THEN) &&
(fra[db.fra_pos].ignore_size > rl[no_of_listed_files].size)))
{
if ((rl[no_of_listed_files].got_date == NO) ||
(fra[db.fra_pos].ignore_file_time == 0))
{
*file_size_to_retrieve += file_size;
*files_to_retrieve += 1;
no_of_listed_files++;
}
else
{
time_t diff_time;
diff_time = current_time - rl[no_of_listed_files].file_mtime;
if (((fra[db.fra_pos].gt_lt_sign & IFTIME_EQUAL) &&
(fra[db.fra_pos].ignore_file_time == diff_time)) ||
((fra[db.fra_pos].gt_lt_sign & IFTIME_LESS_THEN) &&
(fra[db.fra_pos].ignore_file_time < diff_time)) ||
((fra[db.fra_pos].gt_lt_sign & IFTIME_GREATER_THEN) &&
(fra[db.fra_pos].ignore_file_time > diff_time)))
{
*file_size_to_retrieve += file_size;
*files_to_retrieve += 1;
no_of_listed_files++;
}
else
{
return(1);
}
}
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
if ((fra[db.fra_pos].stupid_mode == YES) ||
(fra[db.fra_pos].remove == YES) ||
((*files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
(*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size)))
#else
if ((*files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
(*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size))
#endif
{
rl[no_of_listed_files - 1].assigned = (unsigned char)db.job_no + 1;
}
else
{
rl[no_of_listed_files - 1].assigned = 0;
*file_size_to_retrieve -= file_size;
*files_to_retrieve -= 1;
*more_files_in_list = YES;
}
*(int *)((char *)rl - AFD_WORD_OFFSET) = no_of_listed_files;
#ifdef DEBUG_ASSIGNMENT
trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL,
# if SIZEOF_OFF_T == 4
"%s assigned %d: file_name=%s assigned=%d size=%ld",
# else
"%s assigned %d: file_name=%s assigned=%d size=%lld",
# endif
(fra[db.fra_pos].ls_data_alias[0] == '\0') ? fra[db.fra_pos].dir_alias : fra[db.fra_pos].ls_data_alias,
i, rl[i].file_name, (int)rl[i].assigned, (pri_off_t)rl[i].size);
#endif /* DEBUG_ASSIGNMENT */
return(0);
}
else
{
return(1);
}
}
/*################## get_remote_file_names_ftp_list() ###################*/
int
get_remote_file_names_ftp_list(off_t *file_size_to_retrieve,
int *more_files_in_list)
{
int files_to_retrieve = 0,
i = 0;
*file_size_to_retrieve = 0;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
if ((*more_files_in_list == YES) ||
(db.special_flag & DISTRIBUTED_HELPER_JOB) ||
((db.special_flag & OLD_ERROR_JOB) && (db.retries < 30) &&
(fra[db.fra_pos].stupid_mode != YES) &&
(fra[db.fra_pos].remove != YES)))
#else
if (rl_fd == -1)
{
try_attach_again:
if (attach_ls_data(db.fra_pos, db.fsa_pos, db.special_flag, YES) == INCORRECT)
{
(void)ftp_quit();
exit(INCORRECT);
}
if ((db.special_flag & DISTRIBUTED_HELPER_JOB) &&
((fra[db.fra_pos].stupid_mode == YES) ||
(fra[db.fra_pos].remove == YES)))
{
# ifdef LOCK_DEBUG
if (rlock_region(rl_fd, LOCK_RETR_PROC,
__FILE__, __LINE__) == LOCK_IS_SET)
# else
if (rlock_region(rl_fd, LOCK_RETR_PROC) == LOCK_IS_SET)
# endif
{
if (i == 0)
{
system_log(DEBUG_SIGN, __FILE__, __LINE__,
"Hmm, lock is set. Assume ls_data file was just modified. Lets try it again. (job_no=%d fsa_pos=%d)",
(int)db.job_no, db.fsa_pos);
}
else
{
if (i == 30)
{
trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL,
"Have waited %d seconds, but unable to get a lock. Terminating.",
(i * 100000) / 1000000);
(void)ftp_quit();
exit(SUCCESS);
}
my_usleep(100000L);
}
detach_ls_data(NO);
i++;
goto try_attach_again;
}
}
}
if ((*more_files_in_list == YES) ||
(db.special_flag & DISTRIBUTED_HELPER_JOB) ||
((db.special_flag & OLD_ERROR_JOB) && (db.retries < 30)))
#endif
{
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
if (rl_fd == -1)
{
if (attach_ls_data(db.fra_pos, db.fsa_pos, db.special_flag, YES) == INCORRECT)
{
(void)ftp_quit();
exit(INCORRECT);
}
}
#endif
*more_files_in_list = NO;
for (i = 0; i < no_of_listed_files; i++)
{
if ((rl[i].retrieved == NO) && (rl[i].assigned == 0))
{
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
if ((fra[db.fra_pos].stupid_mode == YES) ||
(fra[db.fra_pos].remove == YES) ||
((files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
(*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size)))
#else
if ((files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
(*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size))
#endif
{
/* Lock this file in list. */
#ifdef LOCK_DEBUG
if (lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i),
__FILE__, __LINE__) == LOCK_IS_NOT_SET)
#else
if (lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i)) == LOCK_IS_NOT_SET)
#endif
{
if ((fra[db.fra_pos].ignore_size == -1) ||
((fra[db.fra_pos].gt_lt_sign & ISIZE_EQUAL) &&
(fra[db.fra_pos].ignore_size == rl[i].size)) ||
((fra[db.fra_pos].gt_lt_sign & ISIZE_LESS_THEN) &&
(fra[db.fra_pos].ignore_size < rl[i].size)) ||
((fra[db.fra_pos].gt_lt_sign & ISIZE_GREATER_THEN) &&
(fra[db.fra_pos].ignore_size > rl[i].size)))
{
if ((rl[i].got_date == NO) ||
(fra[db.fra_pos].ignore_file_time == 0))
{
files_to_retrieve++;
if ((fra[db.fra_pos].stupid_mode == APPEND_ONLY) &&
(rl[i].size > rl[i].prev_size))
{
*file_size_to_retrieve += (rl[i].size - rl[i].prev_size);
}
else
{
*file_size_to_retrieve += rl[i].size;
}
rl[i].assigned = (unsigned char)db.job_no + 1;
}
else
{
time_t diff_time;
diff_time = current_time - rl[i].file_mtime;
if (((fra[db.fra_pos].gt_lt_sign & IFTIME_EQUAL) &&
(fra[db.fra_pos].ignore_file_time == diff_time)) ||
((fra[db.fra_pos].gt_lt_sign & IFTIME_LESS_THEN) &&
(fra[db.fra_pos].ignore_file_time < diff_time)) ||
((fra[db.fra_pos].gt_lt_sign & IFTIME_GREATER_THEN) &&
(fra[db.fra_pos].ignore_file_time > diff_time)))
{
files_to_retrieve++;
if ((fra[db.fra_pos].stupid_mode == APPEND_ONLY) &&
(rl[i].size > rl[i].prev_size))
{
*file_size_to_retrieve += (rl[i].size - rl[i].prev_size);
}
else
{
*file_size_to_retrieve += rl[i].size;
}
rl[i].assigned = (unsigned char)db.job_no + 1;
}
}
#ifdef DEBUG_ASSIGNMENT
trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL,
# if SIZEOF_OFF_T == 4
"%s assigned %d: file_name=%s assigned=%d size=%ld",
# else
"%s assigned %d: file_name=%s assigned=%d size=%lld",
# endif
(fra[db.fra_pos].ls_data_alias[0] == '\0') ? fra[db.fra_pos].dir_alias : fra[db.fra_pos].ls_data_alias,
i, rl[i].file_name, (int)rl[i].assigned,
(pri_off_t)rl[i].size);
#endif /* DEBUG_ASSIGNMENT */
}
#ifdef LOCK_DEBUG
unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i),
__FILE__, __LINE__);
#else
unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i));
#endif
}
}
else
{
*more_files_in_list = YES;
break;
}
}
}
}
else
{
unsigned int files_deleted = 0,
list_length = 0;
int gotcha,
j,
k,
nfg, /* Number of file mask. */
status,
type;
char file_name[MAX_FILENAME_LENGTH + 1],
*list = NULL,
*p_end,
*p_mask,
*p_start;
time_t file_mtime;
off_t file_size,
file_size_deleted = 0,
list_size = 0;
struct ftpparse fp;
struct file_mask *fml = NULL;
struct tm *p_tm;
/*
* Get a directory listing from the remote site so we can see
* what files are there.
*/
#ifdef WITH_SSL
if (db.auth == BOTH)
{
type = LIST_CMD | BUFFERED_LIST | ENCRYPT_DATA;
}
else
{
#endif
type = LIST_CMD | BUFFERED_LIST;
#ifdef WITH_SSL
}
#endif
if ((status = ftp_list(db.mode_flag, type, &list)) != SUCCESS)
{
trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, msg_str,
"Failed to send LIST command (%d).", status);
(void)ftp_quit();
exit(LIST_ERROR);
}
if (list != NULL)
{
/* Get all file masks for this directory. */
if ((j = read_file_mask(fra[db.fra_pos].dir_alias, &nfg, &fml)) == INCORRECT)
{
if (j == LOCKFILE_NOT_THERE)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"Failed to set lock in file masks for %s, because the file is not there.",
fra[db.fra_pos].dir_alias);
}
else if (j == LOCK_IS_SET)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"Failed to get the file masks for %s, because lock is already set",
fra[db.fra_pos].dir_alias);
}
else
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"Failed to get the file masks for %s. (%d)",
fra[db.fra_pos].dir_alias, j);
}
if (fml != NULL)
{
free(fml);
}
(void)ftp_quit();
exit(INCORRECT);
}
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
if ((fra[db.fra_pos].stupid_mode == YES) ||
(fra[db.fra_pos].remove == YES))
{
if (reset_ls_data(db.fra_pos) == INCORRECT)
{
(void)ftp_quit();
exit(INCORRECT);
}
}
else
{
if (rl_fd == -1)
{
if (attach_ls_data(db.fra_pos, db.fsa_pos, db.special_flag,
YES) == INCORRECT)
{
(void)ftp_quit();
exit(INCORRECT);
}
}
}
#else
if (rl_fd == -1)
{
if (attach_ls_data(db.fra_pos, db.fsa_pos, db.special_flag,
YES) == INCORRECT)
{
(void)ftp_quit();
exit(INCORRECT);
}
}
if ((fra[db.fra_pos].stupid_mode == YES) ||
(fra[db.fra_pos].remove == YES))
{
/*
* If all files from the previous listing have been
* collected, lets reset the ls_data structure or otherwise
* it keeps on growing forever.
*/
# ifdef LOCK_DEBUG
if (lock_region(rl_fd, LOCK_RETR_PROC,
__FILE__, __LINE__) == LOCK_IS_NOT_SET)
# else
if (lock_region(rl_fd, LOCK_RETR_PROC) == LOCK_IS_NOT_SET)
# endif
{
if (reset_ls_data(db.fra_pos) == INCORRECT)
{
(void)ftp_quit();
exit(INCORRECT);
}
}
# ifdef LOCK_DEBUG
unlock_region(rl_fd, LOCK_RETR_PROC, __FILE__, __LINE__);
# else
unlock_region(rl_fd, LOCK_RETR_PROC);
# endif
}
#endif
if ((fra[db.fra_pos].ignore_file_time != 0) ||
(fra[db.fra_pos].delete_files_flag & UNKNOWN_FILES))
{
/* Note: FTP returns GMT so we need to convert this to GMT! */
current_time = time(NULL);
p_tm = gmtime(¤t_time);
current_time = mktime(p_tm);
}
/*
* Evaluate the list from the LIST command.
*/
p_end = list;
do
{
p_start = p_end;
while ((*p_end != '\r') && (*p_end != '\n') && (*p_end != '\0'))
{
p_end++;
}
if ((ftpparse(&fp, &file_size, &file_mtime, p_start, p_end - p_start) == 1) &&
((fp.flagtryretr == 1) &&
((fp.name[0] != '.') ||
(fra[db.fra_pos].dir_flag & ACCEPT_DOT_FILES))))
{
list_length++;
list_size += file_size;
if (fp.namelen < MAX_FILENAME_LENGTH)
{
/* Store file name */
(void)memcpy(file_name, fp.name, fp.namelen);
file_name[fp.namelen] = '\0';
if (fra[db.fra_pos].dir_flag == ALL_DISABLED)
{
delete_remote_file(FTP, file_name, fp.namelen,
#ifdef _DELETE_LOG
DELETE_HOST_DISABLED,
#endif
&files_deleted,
&file_size_deleted, file_size);
}
else
{
gotcha = NO;
for (k = 0; k < nfg; k++)
{
p_mask = fml[k].file_list;
for (j = 0; j < fml[k].fc; j++)
{
if ((status = pmatch(p_mask, file_name, NULL)) == 0)
{
if (check_list(file_name, file_size, file_mtime,
&files_to_retrieve,
file_size_to_retrieve,
more_files_in_list) == 0)
{
gotcha = YES;
}
else
{
gotcha = NEITHER;
}
break;
}
else if (status == 1)
{
/* This file is definitly NOT wanted! */
/* Lets skip the rest of this group. */
break;
}
#ifdef SHOW_FILTER_MISSES
if ((status == -1) ||
(fsa->debug > NORMAL_MODE))
{
char tmp_mask[MAX_FILENAME_LENGTH];
if (expand_filter(p_mask, tmp_mask, time(NULL)) == YES)
{
trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL,
"%s (%s) not fitting %s",
p_mask, tmp_mask, file_name);
}
else
{
trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL,
"%s not fitting %s",
p_mask, file_name);
}
}
#endif
NEXT(p_mask);
}
if ((gotcha == YES) || (gotcha == NEITHER))
{
break;
}
}
if ((gotcha == NO) && (status != 0) &&
(fra[db.fra_pos].delete_files_flag & UNKNOWN_FILES))
{
time_t diff_time = current_time - file_mtime;
if ((fra[db.fra_pos].unknown_file_time == -2) ||
((diff_time > fra[db.fra_pos].unknown_file_time) &&
(diff_time > DEFAULT_TRANSFER_TIMEOUT)))
{
delete_remote_file(FTP, file_name, fp.namelen,
#ifdef _DELETE_LOG
DEL_UNKNOWN_FILE,
#endif
&files_deleted,
&file_size_deleted, file_size);
}
}
}
}
else
{
(void)memcpy(file_name, fp.name, MAX_FILENAME_LENGTH);
file_name[MAX_FILENAME_LENGTH] = '\0';
trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL,
"Remote file name `%s' is to long, it may only be %d bytes long.",
file_name, MAX_FILENAME_LENGTH);
}
}
while ((*p_end == '\r') || (*p_end == '\n'))
{
p_end++;
}
} while (*p_end != '\0');
free(list);
/* Free file mask list. */
for (i = 0; i < nfg; i++)
{
free(fml[i].file_list);
}
free(fml);
}
if (files_deleted > 0)
{
trans_log(DEBUG_SIGN, NULL, 0, NULL, NULL,
#if SIZEOF_OFF_T == 4
"%d files %ld bytes found for retrieving %s[%u files with %ld bytes in %s (deleted %u files with %ld bytes)]. @%x",
#else
"%d files %lld bytes found for retrieving %s[%u files with %lld bytes in %s (deleted %u files with %lld bytes)]. @%x",
#endif
files_to_retrieve, (pri_off_t)(*file_size_to_retrieve),
(*more_files_in_list == YES) ? "(+) " : "",
list_length, (pri_off_t)list_size,
(db.target_dir[0] == '\0') ? "home dir" : db.target_dir,
files_deleted, (pri_off_t)file_size_deleted, db.id.dir);
}
else
{
trans_log(DEBUG_SIGN, NULL, 0, NULL, NULL,
#if SIZEOF_OFF_T == 4
"%d files %ld bytes found for retrieving %s[%u files with %ld bytes in %s]. @%x",
#else
"%d files %lld bytes found for retrieving %s[%u files with %lld bytes in %s]. @%x",
#endif
files_to_retrieve, (pri_off_t)(*file_size_to_retrieve),
(*more_files_in_list == YES) ? "(+) " : "",
list_length, (pri_off_t)list_size,
(db.target_dir[0] == '\0') ? "home dir" : db.target_dir,
db.id.dir);
}
/*
* Remove all files from the remote_list structure that are not
* in the current buffer.
*/
if ((fra[db.fra_pos].stupid_mode != YES) &&
(fra[db.fra_pos].remove == NO))
{
int files_removed = 0,
i;
size_t move_size;
for (i = 0; i < (no_of_listed_files - files_removed); i++)
{
if (rl[i].in_list == NO)
{
int j = i;
while ((rl[j].in_list == NO) &&
(j < (no_of_listed_files - files_removed)))
{
j++;
}
if (j != (no_of_listed_files - files_removed))
{
move_size = (no_of_listed_files - files_removed - j) *
sizeof(struct retrieve_list);
(void)memmove(&rl[i], &rl[j], move_size);
}
files_removed += (j - i);
}
}
if (files_removed > 0)
{
int current_no_of_listed_files = no_of_listed_files;
size_t new_size,
old_size;
no_of_listed_files -= files_removed;
if (no_of_listed_files < 0)
{
system_log(DEBUG_SIGN, __FILE__, __LINE__,
"Hmmm, no_of_listed_files = %d", no_of_listed_files);
no_of_listed_files = 0;
}
if (no_of_listed_files == 0)
{
new_size = (RETRIEVE_LIST_STEP_SIZE * sizeof(struct retrieve_list)) +
AFD_WORD_OFFSET;
}
else
{
new_size = (((no_of_listed_files / RETRIEVE_LIST_STEP_SIZE) + 1) *
RETRIEVE_LIST_STEP_SIZE * sizeof(struct retrieve_list)) +
AFD_WORD_OFFSET;
}
old_size = (((current_no_of_listed_files / RETRIEVE_LIST_STEP_SIZE) + 1) *
RETRIEVE_LIST_STEP_SIZE * sizeof(struct retrieve_list)) +
AFD_WORD_OFFSET;
if (old_size != new_size)
{
char *ptr;
ptr = (char *)rl - AFD_WORD_OFFSET;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
if ((fra[db.fra_pos].stupid_mode == YES) ||
(fra[db.fra_pos].remove == YES))
{
if ((ptr = realloc(ptr, new_size)) == NULL)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"realloc() error : %s", strerror(errno));
(void)ftp_quit();
exit(INCORRECT);
}
}
else
{
#endif
if ((ptr = mmap_resize(rl_fd, ptr, new_size)) == (caddr_t) -1)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"mmap_resize() error : %s", strerror(errno));
(void)ftp_quit();
exit(INCORRECT);
}
rl_size = new_size;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
}
#endif
ptr += AFD_WORD_OFFSET;
rl = (struct retrieve_list *)ptr;
}
*(int *)((char *)rl - AFD_WORD_OFFSET) = no_of_listed_files;
}
}
}
return(files_to_retrieve);
}
/*########################## convert_ls_data() ##########################*/
char *
convert_ls_data(int old_rl_fd,
char *old_rl_file,
off_t *old_rl_size,
int old_no_of_listed_files,
char *old_rl_ptr,
unsigned char old_version,
unsigned char new_version)
{
char *ptr;
if ((old_version == 0) && (new_version == 1))
{
int i;
size_t new_size;
struct retrieve_list_0 *old_rl;
struct retrieve_list_1 *new_rl;
old_rl = (struct retrieve_list_0 *)old_rl_ptr;
new_size = old_no_of_listed_files * sizeof(struct retrieve_list_1);
if ((ptr = malloc(new_size)) == NULL)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"malloc() error [%d %d] : %s",
old_no_of_listed_files, new_size, strerror(errno));
ptr = (char *)old_rl;
ptr -= AFD_WORD_OFFSET_0;
#ifdef HAVE_MMAP
if (munmap(ptr, *old_rl_size) == -1)
#else
if (munmap_emu(ptr) == -1)
#endif
{
system_log(WARN_SIGN, __FILE__, __LINE__,
"Failed to munmap() %s : %s",
old_rl_file, strerror(errno));
}
*old_rl_size = -1;
return(NULL);
}
(void)memset(ptr, 0, new_size);
new_rl = (struct retrieve_list_1 *)ptr;
/*
* Copy all the old data into the new region.
*/
for (i = 0; i < old_no_of_listed_files; i++)
{
(void)strcpy(new_rl[i].file_name, old_rl[i].file_name);
new_rl[i].assigned = 0;
new_rl[i].special_flag = 0;
new_rl[i].got_date = old_rl[i].got_date;
new_rl[i].retrieved = old_rl[i].retrieved;
new_rl[i].in_list = old_rl[i].in_list;
new_rl[i].size = old_rl[i].size;
new_rl[i].file_mtime = old_rl[i].file_mtime;
}
ptr = (char *)old_rl;
ptr -= AFD_WORD_OFFSET_0;
/*
* Resize the old retrieve list to the size of new one and then copy
* the new structure into it. Then update the retrieve list version
* number.
*/
if ((ptr = mmap_resize(old_rl_fd, ptr, new_size + AFD_WORD_OFFSET_1)) == (caddr_t) -1)
{
system_log(ERROR_SIGN, __FILE__, __LINE__,
"Failed to mmap_resize() %s : %s",
old_rl_file, strerror(errno));
free((void *)new_rl);
return(NULL);
}
ptr += AFD_WORD_OFFSET_1;
(void)memcpy(ptr, new_rl, new_size);
free((void *)new_rl);
ptr -= AFD_WORD_OFFSET_1;
*(ptr + SIZEOF_INT + 1 + 1) = 0; /* Not used. */
*(ptr + SIZEOF_INT + 1 + 1 + 1) = new_version;
*(int *)(ptr + SIZEOF_INT + 4) = 0; /* Not used. */
*(ptr + SIZEOF_INT + 4 + SIZEOF_INT) = 0; /* Not used. */
*(ptr + SIZEOF_INT + 4 + SIZEOF_INT + 1) = 0; /* Not used. */
*(ptr + SIZEOF_INT + 4 + SIZEOF_INT + 2) = 0; /* Not used. */
*(ptr + SIZEOF_INT + 4 + SIZEOF_INT + 3) = 0; /* Not used. */
*old_rl_size = new_size + AFD_WORD_OFFSET_1;
system_log(INFO_SIGN, NULL, 0,
"Converted retrieve list for %s from version %d to %d.",
basename(old_rl_file), (int)old_version, (int)new_version);
}
else
{
system_log(ERROR_SIGN, NULL, 0,
"Don't know how to convert a version %d of AFD ls data type to version %d.",
old_version, new_version);
ptr = NULL;
}
return(ptr);
}