Example #1
0
/*############################ convert_jid() ############################*/
char *
convert_jid(int           old_jid_fd,
            char          *old_job_id_data_file,
            size_t        *old_jid_size,
            int           old_no_of_job_ids,
            unsigned char old_version,
            unsigned char new_version)
{
   char        *ptr;
   struct stat stat_buf;

   if ((old_version == 1) && (new_version == 2))
   {
      int                  i;
      size_t               new_size;
      struct job_id_data_1 *old_jid;
      struct job_id_data_2 *new_jid;

      /* Get the size of the old JID file. */
      if (fstat(old_jid_fd, &stat_buf) < 0)
      {
         system_log(ERROR_SIGN, __FILE__, __LINE__,
                    "Failed to fstat() %s : %s",
                    old_job_id_data_file, strerror(errno));
         *old_jid_size = -1;
         return(NULL);
      }
      else
      {
         if (stat_buf.st_size > 0)
         {
#ifdef HAVE_MMAP
            if ((ptr = mmap(NULL, stat_buf.st_size, (PROT_READ | PROT_WRITE),
                            MAP_SHARED, old_jid_fd, 0)) == (caddr_t) -1)
#else
            if ((ptr = mmap_emu(NULL, stat_buf.st_size, (PROT_READ | PROT_WRITE),
                                MAP_SHARED, old_job_id_data_file, 0)) == (caddr_t) -1)
#endif
            {
               system_log(ERROR_SIGN, __FILE__, __LINE__,
                          "Failed to mmap() to %s : %s",
                          old_job_id_data_file, strerror(errno));
               *old_jid_size = -1;
               return(NULL);
            }
         }
         else
         {
            system_log(ERROR_SIGN, __FILE__, __LINE__,
                       "FSA file %s is empty.", old_job_id_data_file);
            *old_jid_size = -1;
            return(NULL);
         }
      }

      ptr += AFD_WORD_OFFSET_1;
      old_jid = (struct job_id_data_1 *)ptr;

      new_size = ((old_no_of_job_ids / JOB_ID_DATA_STEP_SIZE) + 1) *
                 JOB_ID_DATA_STEP_SIZE * sizeof(struct job_id_data_2);
      if ((ptr = malloc(new_size)) == NULL)
      {
         system_log(ERROR_SIGN, __FILE__, __LINE__,
                    "malloc() error [%d %d] : %s",
                    old_no_of_job_ids, new_size, strerror(errno));
         ptr = (char *)old_jid;
         ptr -= AFD_WORD_OFFSET_1;
#ifdef HAVE_MMAP
         if (munmap(ptr, stat_buf.st_size) == -1)
#else
         if (munmap_emu(ptr) == -1)
#endif
         {
            system_log(WARN_SIGN, __FILE__, __LINE__,
                       "Failed to munmap() %s : %s",
                       old_job_id_data_file, strerror(errno));
         }
         *old_jid_size = -1;
         return(NULL);
      }
      (void)memset(ptr, 0, new_size);
      new_jid = (struct job_id_data_2 *)ptr;

      /*
       * Copy all the old data into the new region.
       */
      for (i = 0; i < old_no_of_job_ids; i++)
      {
         (void)strcpy(new_jid[i].host_alias, old_jid[i].host_alias);
         (void)strcpy(new_jid[i].recipient, old_jid[i].recipient);
         (void)memcpy(new_jid[i].soptions, old_jid[i].soptions, MAX_OPTION_LENGTH_1);
         (void)memcpy(new_jid[i].loptions, old_jid[i].loptions, MAX_OPTION_LENGTH_1);
         new_jid[i].no_of_soptions = old_jid[i].no_of_soptions;
         new_jid[i].no_of_loptions = old_jid[i].no_of_loptions;
         new_jid[i].priority       = old_jid[i].priority;
         new_jid[i].dir_id_pos     = old_jid[i].dir_id_pos;
         new_jid[i].dir_config_id  = old_jid[i].dir_config_id;
         new_jid[i].file_mask_id   = old_jid[i].file_mask_id;
         new_jid[i].dir_id         = old_jid[i].dir_id;
         new_jid[i].job_id         = old_jid[i].job_id;
         new_jid[i].host_id        = get_str_checksum(new_jid[i].host_alias);
         new_jid[i].recipient_id   = get_str_checksum(new_jid[i].recipient);
      }

      ptr = (char *)old_jid;
      ptr -= AFD_WORD_OFFSET_1;

      /*
       * Resize the old JID to the size of new one and then copy
       * the new structure into it.
       */
      if ((ptr = mmap_resize(old_jid_fd, ptr, new_size + AFD_WORD_OFFSET_2)) == (caddr_t) -1)
      {
         system_log(ERROR_SIGN, __FILE__, __LINE__,
                    "Failed to mmap_resize() %s : %s",
                    old_job_id_data_file, strerror(errno));
         free((void *)new_jid);
         return(NULL);
      }
      ptr += AFD_WORD_OFFSET_2;
      (void)memcpy(ptr, new_jid, new_size);
      free((void *)new_jid);
      ptr -= AFD_WORD_OFFSET_2;
      *(int *)ptr = old_no_of_job_ids;
      *old_jid_size = new_size + AFD_WORD_OFFSET_2;

      system_log(INFO_SIGN, NULL, 0,
                 "Converted JID from verion %d to %d.",
                 (int)old_version, (int)new_version);
      system_log(DEBUG_SIGN, __FILE__, __LINE__,
#if SIZEOF_OFF_T == 4
# if SIZEOF_SIZE_T == 4
                 "filesize old: %ld (%d) new: %ld (%d)  Number of jobs: %d",
# else
                 "filesize old: %ld (%d) new: %lld (%d)  Number of jobs: %d",
# endif
#else
# if SIZEOF_SIZE_T == 4
                 "filesize old: %lld (%d) new: %ld (%d)  Number of jobs: %d",
# else
                 "filesize old: %lld (%d) new: %lld (%d)  Number of jobs: %d",
# endif
#endif
                 (pri_off_t)stat_buf.st_size, (int)sizeof(struct job_id_data_1),
                 (pri_size_t)new_size + AFD_WORD_OFFSET_2,
                 (int)sizeof(struct job_id_data_2), old_no_of_job_ids);
   }
   else if ((old_version == 1) && (new_version == 3))
        {
           int                  i;
           size_t               new_size;
           struct job_id_data_1 *old_jid;
           struct job_id_data_3 *new_jid;

           /* Get the size of the old JID file. */
           if (fstat(old_jid_fd, &stat_buf) < 0)
           {
              system_log(ERROR_SIGN, __FILE__, __LINE__,
                         "Failed to fstat() %s : %s",
                         old_job_id_data_file, strerror(errno));
              *old_jid_size = -1;
              return(NULL);
           }
           else
           {
              if (stat_buf.st_size > 0)
              {
#ifdef HAVE_MMAP
                 if ((ptr = mmap(NULL, stat_buf.st_size, (PROT_READ | PROT_WRITE),
                                 MAP_SHARED, old_jid_fd, 0)) == (caddr_t) -1)
#else
                 if ((ptr = mmap_emu(NULL, stat_buf.st_size, (PROT_READ | PROT_WRITE),
                                     MAP_SHARED, old_job_id_data_file, 0)) == (caddr_t) -1)
#endif
                 {
                    system_log(ERROR_SIGN, __FILE__, __LINE__,
                               "Failed to mmap() to %s : %s",
                                    old_job_id_data_file, strerror(errno));
                    *old_jid_size = -1;
                    return(NULL);
                 }
              }
              else
              {
                 system_log(ERROR_SIGN, __FILE__, __LINE__,
                            "FSA file %s is empty.", old_job_id_data_file);
                 *old_jid_size = -1;
                 return(NULL);
              }
           }

           ptr += AFD_WORD_OFFSET_1;
           old_jid = (struct job_id_data_1 *)ptr;

           new_size = ((old_no_of_job_ids / JOB_ID_DATA_STEP_SIZE) + 1) *
                      JOB_ID_DATA_STEP_SIZE * sizeof(struct job_id_data_3);
           if ((ptr = malloc(new_size)) == NULL)
           {
              system_log(ERROR_SIGN, __FILE__, __LINE__,
                         "malloc() error [%d %d] : %s",
                         old_no_of_job_ids, new_size, strerror(errno));
              ptr = (char *)old_jid;
              ptr -= AFD_WORD_OFFSET_1;
#ifdef HAVE_MMAP
              if (munmap(ptr, stat_buf.st_size) == -1)
#else
              if (munmap_emu(ptr) == -1)
#endif
              {
                 system_log(WARN_SIGN, __FILE__, __LINE__,
                            "Failed to munmap() %s : %s",
                            old_job_id_data_file, strerror(errno));
              }
              *old_jid_size = -1;
              return(NULL);
           }
           (void)memset(ptr, 0, new_size);
           new_jid = (struct job_id_data_3 *)ptr;

           /*
            * Copy all the old data into the new region.
            */
           for (i = 0; i < old_no_of_job_ids; i++)
           {
              (void)strcpy(new_jid[i].host_alias, old_jid[i].host_alias);
              (void)strcpy(new_jid[i].recipient, old_jid[i].recipient);
              (void)memcpy(new_jid[i].soptions, old_jid[i].soptions, MAX_OPTION_LENGTH_1);
              (void)memcpy(new_jid[i].loptions, old_jid[i].loptions, MAX_OPTION_LENGTH_1);
              new_jid[i].creation_time  = 0L;
              new_jid[i].special_flag   = 0;
              new_jid[i].no_of_soptions = old_jid[i].no_of_soptions;
              new_jid[i].no_of_loptions = old_jid[i].no_of_loptions;
              new_jid[i].priority       = old_jid[i].priority;
              new_jid[i].dir_id_pos     = old_jid[i].dir_id_pos;
              new_jid[i].dir_config_id  = old_jid[i].dir_config_id;
              new_jid[i].file_mask_id   = old_jid[i].file_mask_id;
              new_jid[i].dir_id         = old_jid[i].dir_id;
              new_jid[i].job_id         = old_jid[i].job_id;
              new_jid[i].host_id        = get_str_checksum(new_jid[i].host_alias);
              new_jid[i].recipient_id   = get_str_checksum(new_jid[i].recipient);
           }

           ptr = (char *)old_jid;
           ptr -= AFD_WORD_OFFSET_1;

           /*
            * Resize the old JID to the size of new one and then copy
            * the new structure into it.
            */
           if ((ptr = mmap_resize(old_jid_fd, ptr, new_size + AFD_WORD_OFFSET_3)) == (caddr_t) -1)
           {
              system_log(ERROR_SIGN, __FILE__, __LINE__,
                         "Failed to mmap_resize() %s : %s",
                         old_job_id_data_file, strerror(errno));
              free((void *)new_jid);
              return(NULL);
           }
           ptr += AFD_WORD_OFFSET_3;
           (void)memcpy(ptr, new_jid, new_size);
           free((void *)new_jid);
           ptr -= AFD_WORD_OFFSET_3;
           *(int *)ptr = old_no_of_job_ids;
           *old_jid_size = new_size + AFD_WORD_OFFSET_3;

           system_log(INFO_SIGN, NULL, 0,
                      "Converted JID from verion %d to %d.",
                      (int)old_version, (int)new_version);
           system_log(DEBUG_SIGN, __FILE__, __LINE__,
#if SIZEOF_OFF_T == 4
# if SIZEOF_SIZE_T == 4
                      "filesize old: %ld (%d) new: %ld (%d)  Number of jobs: %d",
# else
                      "filesize old: %ld (%d) new: %lld (%d)  Number of jobs: %d",
# endif
#else
# if SIZEOF_SIZE_T == 4
                      "filesize old: %lld (%d) new: %ld (%d)  Number of jobs: %d",
# else
                      "filesize old: %lld (%d) new: %lld (%d)  Number of jobs: %d",
# endif
#endif
                      (pri_off_t)stat_buf.st_size, (int)sizeof(struct job_id_data_1),
                      (pri_size_t)new_size + AFD_WORD_OFFSET_3,
                      (int)sizeof(struct job_id_data_3), old_no_of_job_ids);
        }
   else if ((old_version == 2) && (new_version == 3))
        {
           int                  i;
           size_t               new_size;
           struct job_id_data_2 *old_jid;
           struct job_id_data_3 *new_jid;

           /* Get the size of the old JID file. */
           if (fstat(old_jid_fd, &stat_buf) < 0)
           {
              system_log(ERROR_SIGN, __FILE__, __LINE__,
                         "Failed to fstat() %s : %s",
                         old_job_id_data_file, strerror(errno));
              *old_jid_size = -1;
              return(NULL);
           }
           else
           {
              if (stat_buf.st_size > 0)
              {
#ifdef HAVE_MMAP
                 if ((ptr = mmap(NULL, stat_buf.st_size, (PROT_READ | PROT_WRITE),
                                 MAP_SHARED, old_jid_fd, 0)) == (caddr_t) -1)
#else
                 if ((ptr = mmap_emu(NULL, stat_buf.st_size, (PROT_READ | PROT_WRITE),
                                     MAP_SHARED, old_job_id_data_file, 0)) == (caddr_t) -1)
#endif
                 {
                    system_log(ERROR_SIGN, __FILE__, __LINE__,
                               "Failed to mmap() to %s : %s",
                                    old_job_id_data_file, strerror(errno));
                    *old_jid_size = -1;
                    return(NULL);
                 }
              }
              else
              {
                 system_log(ERROR_SIGN, __FILE__, __LINE__,
                            "FSA file %s is empty.", old_job_id_data_file);
                 *old_jid_size = -1;
                 return(NULL);
              }
           }

           ptr += AFD_WORD_OFFSET_2;
           old_jid = (struct job_id_data_2 *)ptr;

           new_size = ((old_no_of_job_ids / JOB_ID_DATA_STEP_SIZE) + 1) *
                      JOB_ID_DATA_STEP_SIZE * sizeof(struct job_id_data_3);
           if ((ptr = malloc(new_size)) == NULL)
           {
              system_log(ERROR_SIGN, __FILE__, __LINE__,
                         "malloc() error [%d %d] : %s",
                         old_no_of_job_ids, new_size, strerror(errno));
              ptr = (char *)old_jid;
              ptr -= AFD_WORD_OFFSET_2;
#ifdef HAVE_MMAP
              if (munmap(ptr, stat_buf.st_size) == -1)
#else
              if (munmap_emu(ptr) == -1)
#endif
              {
                 system_log(WARN_SIGN, __FILE__, __LINE__,
                            "Failed to munmap() %s : %s",
                            old_job_id_data_file, strerror(errno));
              }
              *old_jid_size = -1;
              return(NULL);
           }
           (void)memset(ptr, 0, new_size);
           new_jid = (struct job_id_data_3 *)ptr;

           /*
            * Copy all the old data into the new region.
            */
           for (i = 0; i < old_no_of_job_ids; i++)
           {
              (void)strcpy(new_jid[i].host_alias, old_jid[i].host_alias);
              (void)strcpy(new_jid[i].recipient, old_jid[i].recipient);
              (void)memcpy(new_jid[i].soptions, old_jid[i].soptions, MAX_OPTION_LENGTH_2);
              (void)memcpy(new_jid[i].loptions, old_jid[i].loptions, MAX_OPTION_LENGTH_2);
              new_jid[i].creation_time  = 0L;
              new_jid[i].special_flag   = 0;
              new_jid[i].no_of_soptions = old_jid[i].no_of_soptions;
              new_jid[i].no_of_loptions = old_jid[i].no_of_loptions;
              new_jid[i].priority       = old_jid[i].priority;
              new_jid[i].dir_id_pos     = old_jid[i].dir_id_pos;
              new_jid[i].dir_config_id  = old_jid[i].dir_config_id;
              new_jid[i].file_mask_id   = old_jid[i].file_mask_id;
              new_jid[i].dir_id         = old_jid[i].dir_id;
              new_jid[i].job_id         = old_jid[i].job_id;
              new_jid[i].host_id        = old_jid[i].host_id;
              new_jid[i].recipient_id   = old_jid[i].recipient_id;
           }

           ptr = (char *)old_jid;
           ptr -= AFD_WORD_OFFSET_2;

           /*
            * Resize the old JID to the size of new one and then copy
            * the new structure into it.
            */
           if ((ptr = mmap_resize(old_jid_fd, ptr, new_size + AFD_WORD_OFFSET_3)) == (caddr_t) -1)
           {
              system_log(ERROR_SIGN, __FILE__, __LINE__,
                         "Failed to mmap_resize() %s : %s",
                         old_job_id_data_file, strerror(errno));
              free((void *)new_jid);
              return(NULL);
           }
           ptr += AFD_WORD_OFFSET_3;
           (void)memcpy(ptr, new_jid, new_size);
           free((void *)new_jid);
           ptr -= AFD_WORD_OFFSET_3;
           *(int *)ptr = old_no_of_job_ids;
           *old_jid_size = new_size + AFD_WORD_OFFSET_3;

           system_log(INFO_SIGN, NULL, 0,
                      "Converted JID from verion %d to %d.",
                      (int)old_version, (int)new_version);
           system_log(DEBUG_SIGN, __FILE__, __LINE__,
#if SIZEOF_OFF_T == 4
# if SIZEOF_SIZE_T == 4
                      "filesize old: %ld (%d) new: %ld (%d)  Number of jobs: %d",
# else
                      "filesize old: %ld (%d) new: %lld (%d)  Number of jobs: %d",
# endif
#else
# if SIZEOF_SIZE_T == 4
                      "filesize old: %lld (%d) new: %ld (%d)  Number of jobs: %d",
# else
                      "filesize old: %lld (%d) new: %lld (%d)  Number of jobs: %d",
# endif
#endif
                      (pri_off_t)stat_buf.st_size, (int)sizeof(struct job_id_data_2),
                      (pri_size_t)new_size + AFD_WORD_OFFSET_3,
                      (int)sizeof(struct job_id_data_3), old_no_of_job_ids);
        }
        else
        {
           system_log(ERROR_SIGN, NULL, 0,
                      "Don't know how to convert a version %d JID to version %d.",
                      old_version, new_version);
           ptr = NULL;
        }

   return(ptr);
}
/*+++++++++++++++++++++++++++++ check_list() ++++++++++++++++++++++++++++*/
static int
check_list(char   *file,
           off_t  file_size,
           time_t file_mtime,
           int    *files_to_retrieve,
           off_t  *file_size_to_retrieve,
           int    *more_files_in_list)
{
   int i;

   if ((fra[db.fra_pos].stupid_mode == YES) ||
       (fra[db.fra_pos].remove == YES))
   {
      for (i = 0; i < no_of_listed_files; i++)
      {
         if (CHECK_STRCMP(rl[i].file_name, file) == 0)
         {
            rl[i].in_list = YES;
            if (((rl[i].assigned == 0) || (rl[i].retrieved == YES)) &&
                (((db.special_flag & OLD_ERROR_JOB) == 0) ||
#ifdef LOCK_DEBUG
                   (lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i),
                                __FILE__, __LINE__) == LOCK_IS_NOT_SET)
#else
                   (lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i)) == LOCK_IS_NOT_SET)
#endif
                  ))
            {
               int ret;

               rl[i].file_mtime = file_mtime;
               rl[i].got_date = YES;
               rl[i].size = file_size;
               rl[i].prev_size = 0;

               if ((fra[db.fra_pos].ignore_size == -1) ||
                   ((fra[db.fra_pos].gt_lt_sign & ISIZE_EQUAL) &&
                    (fra[db.fra_pos].ignore_size == rl[i].size)) ||
                   ((fra[db.fra_pos].gt_lt_sign & ISIZE_LESS_THEN) &&
                    (fra[db.fra_pos].ignore_size < rl[i].size)) ||
                   ((fra[db.fra_pos].gt_lt_sign & ISIZE_GREATER_THEN) &&
                    (fra[db.fra_pos].ignore_size > rl[i].size)))
               {
                  if (fra[db.fra_pos].ignore_file_time == 0)
                  {
                     *file_size_to_retrieve += rl[i].size;
                     *files_to_retrieve += 1;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
                     if ((fra[db.fra_pos].stupid_mode == YES) ||
                         (fra[db.fra_pos].remove == YES) ||
                         ((*files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
                          (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size)))
#else
                     if ((*files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
                         (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size))
#endif
                     {
                        rl[i].retrieved = NO;
                        rl[i].assigned = (unsigned char)db.job_no + 1;
                     }
                     else
                     {
                        rl[i].assigned = 0;
                        *file_size_to_retrieve -= rl[i].size;
                        *files_to_retrieve -= 1;
                        *more_files_in_list = YES;
                     }
                     ret = 0;
                  }
                  else
                  {
                     time_t diff_time;

                     diff_time = current_time - rl[i].file_mtime;
                     if (((fra[db.fra_pos].gt_lt_sign & IFTIME_EQUAL) &&
                          (fra[db.fra_pos].ignore_file_time == diff_time)) ||
                         ((fra[db.fra_pos].gt_lt_sign & IFTIME_LESS_THEN) &&
                          (fra[db.fra_pos].ignore_file_time < diff_time)) ||
                         ((fra[db.fra_pos].gt_lt_sign & IFTIME_GREATER_THEN) &&
                          (fra[db.fra_pos].ignore_file_time > diff_time)))
                     {
                        *file_size_to_retrieve += rl[i].size;
                        *files_to_retrieve += 1;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
                        if ((fra[db.fra_pos].stupid_mode == YES) ||
                            (fra[db.fra_pos].remove == YES) ||
                            ((*files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
                             (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size)))
#else
                        if ((*files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
                            (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size))
#endif
                        {
                           rl[i].retrieved = NO;
                           rl[i].assigned = (unsigned char)db.job_no + 1;
                        }
                        else
                        {
                           rl[i].assigned = 0;
                           *file_size_to_retrieve -= rl[i].size;
                           *files_to_retrieve -= 1;
                           *more_files_in_list = YES;
                        }
                        ret = 0;
                     }
                     else
                     {
                        ret = 1;
                     }
                  }
#ifdef DEBUG_ASSIGNMENT
                  trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL,
# if SIZEOF_OFF_T == 4
                            "%s assigned %d: file_name=%s assigned=%d size=%ld",
# else
                            "%s assigned %d: file_name=%s assigned=%d size=%lld",
# endif
                            (fra[db.fra_pos].ls_data_alias[0] == '\0') ? fra[db.fra_pos].dir_alias : fra[db.fra_pos].ls_data_alias,
                            i, rl[i].file_name, (int)rl[i].assigned,
                            (pri_off_t)rl[i].size);
#endif /* DEBUG_ASSIGNMENT */
               }
               else
               {
                  ret = 1;
               }
               if (db.special_flag & OLD_ERROR_JOB)
               {
#ifdef LOCK_DEBUG
                  unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i),
                                __FILE__, __LINE__);
#else
                  unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i));
#endif
               }
               return(ret);
            }
            else
            {
               return(1);
            }
         }
      } /* for (i = 0; i < no_of_listed_files; i++) */
   }
   else
   {
      /* Check if this file is in the list. */
      for (i = 0; i < no_of_listed_files; i++)
      {
         if (CHECK_STRCMP(rl[i].file_name, file) == 0)
         {
            rl[i].in_list = YES;
            if ((rl[i].assigned != 0) ||
                ((fra[db.fra_pos].stupid_mode == GET_ONCE_ONLY) &&
                 (rl[i].retrieved == YES)))
            {
               return(1);
            }

            if (((db.special_flag & OLD_ERROR_JOB) == 0) ||
#ifdef LOCK_DEBUG
                (lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i),
                             __FILE__, __LINE__) == LOCK_IS_NOT_SET)
#else
                (lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i)) == LOCK_IS_NOT_SET)
#endif
               )
            {
               int   ret;
               off_t prev_size = 0;

               if (rl[i].file_mtime != file_mtime)
               {
                  rl[i].file_mtime = file_mtime;
                  rl[i].retrieved = NO;
                  rl[i].assigned = 0;
               }
               rl[i].got_date = YES;
               if (rl[i].size != file_size)
               {
                  prev_size = rl[i].size;
                  rl[i].size = file_size;
                  rl[i].retrieved = NO;
                  rl[i].assigned = 0;
               }
               if (rl[i].retrieved == NO)
               {
                  if ((fra[db.fra_pos].ignore_size == -1) ||
                      ((fra[db.fra_pos].gt_lt_sign & ISIZE_EQUAL) &&
                       (fra[db.fra_pos].ignore_size == rl[i].size)) ||
                      ((fra[db.fra_pos].gt_lt_sign & ISIZE_LESS_THEN) &&
                       (fra[db.fra_pos].ignore_size < rl[i].size)) ||
                      ((fra[db.fra_pos].gt_lt_sign & ISIZE_GREATER_THEN) &&
                       (fra[db.fra_pos].ignore_size > rl[i].size)))
                  {
                     off_t size_to_retrieve;

                     if ((rl[i].got_date == NO) ||
                         (fra[db.fra_pos].ignore_file_time == 0))
                     {
                        if ((fra[db.fra_pos].stupid_mode == APPEND_ONLY) &&
                            (rl[i].size > prev_size))
                        {
                           size_to_retrieve = rl[i].size - prev_size;
                        }
                        else
                        {
                           size_to_retrieve = rl[i].size;
                        }
                        rl[i].prev_size = prev_size;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
                        if ((fra[db.fra_pos].stupid_mode == YES) ||
                            (fra[db.fra_pos].remove == YES) ||
                            (((*files_to_retrieve + 1) < fra[db.fra_pos].max_copied_files) &&
                             ((*file_size_to_retrieve + size_to_retrieve) < fra[db.fra_pos].max_copied_file_size)))
#else
                        if (((*files_to_retrieve + 1) < fra[db.fra_pos].max_copied_files) &&
                            ((*file_size_to_retrieve + size_to_retrieve) < fra[db.fra_pos].max_copied_file_size))
#endif
                        {
                           rl[i].assigned = (unsigned char)db.job_no + 1;
                           *file_size_to_retrieve += size_to_retrieve;
                           *files_to_retrieve += 1;
                        }
                        else
                        {
                           rl[i].assigned = 0;
                           *more_files_in_list = YES;
                        }
                        ret = 0;
                     }
                     else
                     {
                        time_t diff_time;

                        diff_time = current_time - rl[i].file_mtime;
                        if (((fra[db.fra_pos].gt_lt_sign & IFTIME_EQUAL) &&
                             (fra[db.fra_pos].ignore_file_time == diff_time)) ||
                            ((fra[db.fra_pos].gt_lt_sign & IFTIME_LESS_THEN) &&
                             (fra[db.fra_pos].ignore_file_time < diff_time)) ||
                            ((fra[db.fra_pos].gt_lt_sign & IFTIME_GREATER_THEN) &&
                             (fra[db.fra_pos].ignore_file_time > diff_time)))
                        {
                           if ((fra[db.fra_pos].stupid_mode == APPEND_ONLY) &&
                               (rl[i].size > prev_size))
                           {
                              size_to_retrieve = rl[i].size - prev_size;
                           }
                           else
                           {
                              size_to_retrieve = rl[i].size;
                           }
                           rl[i].prev_size = prev_size;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
                           if ((fra[db.fra_pos].stupid_mode == YES) ||
                               (fra[db.fra_pos].remove == YES) ||
                               (((*files_to_retrieve + 1) < fra[db.fra_pos].max_copied_files) &&
                                ((*file_size_to_retrieve + size_to_retrieve) < fra[db.fra_pos].max_copied_file_size)))
#else
                           if (((*files_to_retrieve + 1) < fra[db.fra_pos].max_copied_files) &&
                               ((*file_size_to_retrieve + size_to_retrieve) < fra[db.fra_pos].max_copied_file_size))
#endif
                           {
                              rl[i].assigned = (unsigned char)db.job_no + 1;
                              *file_size_to_retrieve += size_to_retrieve;
                              *files_to_retrieve += 1;
                           }
                           else
                           {
                              rl[i].assigned = 0;
                              *more_files_in_list = YES;
                           }
                           ret = 0;
                        }
                        else
                        {
                           ret = 1;
                        }
                     }
#ifdef DEBUG_ASSIGNMENT
                     trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL,
# if SIZEOF_OFF_T == 4
                               "%s assigned %d: file_name=%s assigned=%d size=%ld",
# else
                               "%s assigned %d: file_name=%s assigned=%d size=%lld",
# endif
                               (fra[db.fra_pos].ls_data_alias[0] == '\0') ? fra[db.fra_pos].dir_alias : fra[db.fra_pos].ls_data_alias,
                               i, rl[i].file_name, (int)rl[i].assigned,
                               (pri_off_t)rl[i].size);
#endif /* DEBUG_ASSIGNMENT */
                  }
                  else
                  {
                     ret = 1;
                  }
               }
               else
               {
                  ret = 1;
               }
               if (db.special_flag & OLD_ERROR_JOB)
               {
#ifdef LOCK_DEBUG
                  unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i),
                                __FILE__, __LINE__);
#else
                  unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i));
#endif
               }
               return(ret);
            }
            else
            {
               return(1);
            }
         }
      } /* for (i = 0; i < no_of_listed_files; i++) */
   }

   /* Add this file to the list. */
   if ((no_of_listed_files != 0) &&
       ((no_of_listed_files % RETRIEVE_LIST_STEP_SIZE) == 0))
   {
      char   *ptr;
      size_t new_size = (((no_of_listed_files / RETRIEVE_LIST_STEP_SIZE) + 1) *
                         RETRIEVE_LIST_STEP_SIZE * sizeof(struct retrieve_list)) +
                         AFD_WORD_OFFSET;

      ptr = (char *)rl - AFD_WORD_OFFSET;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
      if ((fra[db.fra_pos].stupid_mode == YES) ||
          (fra[db.fra_pos].remove == YES))
      {
         if ((ptr = realloc(ptr, new_size)) == NULL)
         {
            system_log(ERROR_SIGN, __FILE__, __LINE__,
                       "realloc() error : %s", strerror(errno));
            (void)ftp_quit();
            exit(INCORRECT);
         }
      }
      else
      {
#endif
         if ((ptr = mmap_resize(rl_fd, ptr, new_size)) == (caddr_t) -1)
         {
            system_log(ERROR_SIGN, __FILE__, __LINE__,
                       "mmap_resize() error : %s", strerror(errno));
            (void)ftp_quit();
            exit(INCORRECT);
         }
         rl_size = new_size;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
      }
#endif
      if (no_of_listed_files < 0)
      {
         system_log(DEBUG_SIGN, __FILE__, __LINE__,
                    "Hmmm, no_of_listed_files = %d", no_of_listed_files);
         no_of_listed_files = 0;
      }
      *(int *)ptr = no_of_listed_files;
      ptr += AFD_WORD_OFFSET;
      rl = (struct retrieve_list *)ptr;
   }
   (void)strcpy(rl[no_of_listed_files].file_name, file);
   rl[no_of_listed_files].retrieved = NO;
   rl[no_of_listed_files].in_list = YES;
   rl[no_of_listed_files].size = file_size;
   rl[no_of_listed_files].prev_size = 0;
   rl[no_of_listed_files].file_mtime = file_mtime;
   rl[no_of_listed_files].got_date = YES;

   if ((fra[db.fra_pos].ignore_size == -1) ||
       ((fra[db.fra_pos].gt_lt_sign & ISIZE_EQUAL) &&
        (fra[db.fra_pos].ignore_size == rl[no_of_listed_files].size)) ||
       ((fra[db.fra_pos].gt_lt_sign & ISIZE_LESS_THEN) &&
        (fra[db.fra_pos].ignore_size < rl[no_of_listed_files].size)) ||
       ((fra[db.fra_pos].gt_lt_sign & ISIZE_GREATER_THEN) &&
        (fra[db.fra_pos].ignore_size > rl[no_of_listed_files].size)))
   {
      if ((rl[no_of_listed_files].got_date == NO) ||
          (fra[db.fra_pos].ignore_file_time == 0))
      {
         *file_size_to_retrieve += file_size;
         *files_to_retrieve += 1;
         no_of_listed_files++;
      }
      else
      {
         time_t diff_time;

         diff_time = current_time - rl[no_of_listed_files].file_mtime;
         if (((fra[db.fra_pos].gt_lt_sign & IFTIME_EQUAL) &&
              (fra[db.fra_pos].ignore_file_time == diff_time)) ||
             ((fra[db.fra_pos].gt_lt_sign & IFTIME_LESS_THEN) &&
              (fra[db.fra_pos].ignore_file_time < diff_time)) ||
             ((fra[db.fra_pos].gt_lt_sign & IFTIME_GREATER_THEN) &&
                 (fra[db.fra_pos].ignore_file_time > diff_time)))
         {
            *file_size_to_retrieve += file_size;
            *files_to_retrieve += 1;
            no_of_listed_files++;
         }
         else
         {
            return(1);
         }
      }
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
      if ((fra[db.fra_pos].stupid_mode == YES) ||
          (fra[db.fra_pos].remove == YES) ||
          ((*files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
           (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size)))
#else
      if ((*files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
          (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size))
#endif
      {
         rl[no_of_listed_files - 1].assigned = (unsigned char)db.job_no + 1;
      }
      else
      {
         rl[no_of_listed_files - 1].assigned = 0;
         *file_size_to_retrieve -= file_size;
         *files_to_retrieve -= 1;
         *more_files_in_list = YES;
      }
      *(int *)((char *)rl - AFD_WORD_OFFSET) = no_of_listed_files;
#ifdef DEBUG_ASSIGNMENT
      trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL,
# if SIZEOF_OFF_T == 4
                "%s assigned %d: file_name=%s assigned=%d size=%ld",
# else
                "%s assigned %d: file_name=%s assigned=%d size=%lld",
# endif
                (fra[db.fra_pos].ls_data_alias[0] == '\0') ? fra[db.fra_pos].dir_alias : fra[db.fra_pos].ls_data_alias,
                i, rl[i].file_name, (int)rl[i].assigned, (pri_off_t)rl[i].size);
#endif /* DEBUG_ASSIGNMENT */
      return(0);
   }
   else
   {
      return(1);
   }
}
/*################## get_remote_file_names_ftp_list() ###################*/
int
get_remote_file_names_ftp_list(off_t *file_size_to_retrieve,
                               int   *more_files_in_list)
{
   int              files_to_retrieve = 0,
                    i = 0;

   *file_size_to_retrieve = 0;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
   if ((*more_files_in_list == YES) ||
       (db.special_flag & DISTRIBUTED_HELPER_JOB) ||
       ((db.special_flag & OLD_ERROR_JOB) && (db.retries < 30) &&
        (fra[db.fra_pos].stupid_mode != YES) &&
        (fra[db.fra_pos].remove != YES)))
#else
   if (rl_fd == -1)
   {
try_attach_again:
      if (attach_ls_data(db.fra_pos, db.fsa_pos, db.special_flag, YES) == INCORRECT)
      {
         (void)ftp_quit();
         exit(INCORRECT);
      }
      if ((db.special_flag & DISTRIBUTED_HELPER_JOB) &&
          ((fra[db.fra_pos].stupid_mode == YES) ||
           (fra[db.fra_pos].remove == YES)))
      {
# ifdef LOCK_DEBUG
         if (rlock_region(rl_fd, LOCK_RETR_PROC,
                          __FILE__, __LINE__) == LOCK_IS_SET)
# else
         if (rlock_region(rl_fd, LOCK_RETR_PROC) == LOCK_IS_SET)
# endif
         {
            if (i == 0)
            {
               system_log(DEBUG_SIGN, __FILE__, __LINE__,
                          "Hmm, lock is set. Assume ls_data file was just modified. Lets try it again. (job_no=%d fsa_pos=%d)",
                          (int)db.job_no, db.fsa_pos);
            }
            else
            {
               if (i == 30)
               {
                  trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL,
                            "Have waited %d seconds, but unable to get a lock. Terminating.",
                            (i * 100000) / 1000000);
                  (void)ftp_quit();
                  exit(SUCCESS);
               }
               my_usleep(100000L);
            }
            detach_ls_data(NO);
            i++;
            goto try_attach_again;
         }
      }
   }

   if ((*more_files_in_list == YES) ||
       (db.special_flag & DISTRIBUTED_HELPER_JOB) ||
       ((db.special_flag & OLD_ERROR_JOB) && (db.retries < 30)))
#endif
   {
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
      if (rl_fd == -1)
      {
         if (attach_ls_data(db.fra_pos, db.fsa_pos, db.special_flag, YES) == INCORRECT)
         {
            (void)ftp_quit();
            exit(INCORRECT);
         }
      }
#endif
      *more_files_in_list = NO;
      for (i = 0; i < no_of_listed_files; i++)
      {
         if ((rl[i].retrieved == NO) && (rl[i].assigned == 0))
         {
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
            if ((fra[db.fra_pos].stupid_mode == YES) ||
                (fra[db.fra_pos].remove == YES) ||
                ((files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
                 (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size)))
#else
            if ((files_to_retrieve < fra[db.fra_pos].max_copied_files) &&
                (*file_size_to_retrieve < fra[db.fra_pos].max_copied_file_size))
#endif
            {
               /* Lock this file in list. */
#ifdef LOCK_DEBUG
               if (lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i),
                               __FILE__, __LINE__) == LOCK_IS_NOT_SET)
#else
               if (lock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i)) == LOCK_IS_NOT_SET)
#endif
               {
                  if ((fra[db.fra_pos].ignore_size == -1) ||
                      ((fra[db.fra_pos].gt_lt_sign & ISIZE_EQUAL) &&
                       (fra[db.fra_pos].ignore_size == rl[i].size)) ||
                      ((fra[db.fra_pos].gt_lt_sign & ISIZE_LESS_THEN) &&
                       (fra[db.fra_pos].ignore_size < rl[i].size)) ||
                      ((fra[db.fra_pos].gt_lt_sign & ISIZE_GREATER_THEN) &&
                       (fra[db.fra_pos].ignore_size > rl[i].size)))
                  {
                     if ((rl[i].got_date == NO) ||
                         (fra[db.fra_pos].ignore_file_time == 0))
                     {
                        files_to_retrieve++;
                        if ((fra[db.fra_pos].stupid_mode == APPEND_ONLY) &&
                            (rl[i].size > rl[i].prev_size))
                        {
                           *file_size_to_retrieve += (rl[i].size - rl[i].prev_size);
                        }
                        else
                        {
                           *file_size_to_retrieve += rl[i].size;
                        }
                        rl[i].assigned = (unsigned char)db.job_no + 1;
                     }
                     else
                     {
                        time_t diff_time;

                        diff_time = current_time - rl[i].file_mtime;
                        if (((fra[db.fra_pos].gt_lt_sign & IFTIME_EQUAL) &&
                             (fra[db.fra_pos].ignore_file_time == diff_time)) ||
                            ((fra[db.fra_pos].gt_lt_sign & IFTIME_LESS_THEN) &&
                             (fra[db.fra_pos].ignore_file_time < diff_time)) ||
                            ((fra[db.fra_pos].gt_lt_sign & IFTIME_GREATER_THEN) &&
                             (fra[db.fra_pos].ignore_file_time > diff_time)))
                        {
                           files_to_retrieve++;
                           if ((fra[db.fra_pos].stupid_mode == APPEND_ONLY) &&
                               (rl[i].size > rl[i].prev_size))
                           {
                              *file_size_to_retrieve += (rl[i].size - rl[i].prev_size);
                           }
                           else
                           {
                              *file_size_to_retrieve += rl[i].size;
                           }
                           rl[i].assigned = (unsigned char)db.job_no + 1;
                        }
                     }
#ifdef DEBUG_ASSIGNMENT
                     trans_log(DEBUG_SIGN, __FILE__, __LINE__, NULL, NULL,
# if SIZEOF_OFF_T == 4
                               "%s assigned %d: file_name=%s assigned=%d size=%ld",
# else
                               "%s assigned %d: file_name=%s assigned=%d size=%lld",
# endif
                               (fra[db.fra_pos].ls_data_alias[0] == '\0') ? fra[db.fra_pos].dir_alias : fra[db.fra_pos].ls_data_alias,
                               i, rl[i].file_name, (int)rl[i].assigned,
                               (pri_off_t)rl[i].size);
#endif /* DEBUG_ASSIGNMENT */
                  }
#ifdef LOCK_DEBUG
                  unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i),
                                __FILE__, __LINE__);
#else
                  unlock_region(rl_fd, (off_t)(LOCK_RETR_FILE + i));
#endif
               }
            }
            else
            {
               *more_files_in_list = YES;
               break;
            }
         }
      }
   }
   else
   {
      unsigned int     files_deleted = 0,
                       list_length = 0;
      int              gotcha,
                       j,
                       k,
                       nfg,           /* Number of file mask. */
                       status,
                       type;
      char             file_name[MAX_FILENAME_LENGTH + 1],
                       *list = NULL,
                       *p_end,
                       *p_mask,
                       *p_start;
      time_t           file_mtime;
      off_t            file_size,
                       file_size_deleted = 0,
                       list_size = 0;
      struct ftpparse  fp;
      struct file_mask *fml = NULL;
      struct tm        *p_tm;

      /*
       * Get a directory listing from the remote site so we can see
       * what files are there.
       */
#ifdef WITH_SSL
      if (db.auth == BOTH)
      {
         type = LIST_CMD | BUFFERED_LIST | ENCRYPT_DATA;
      }
      else
      {
#endif
         type = LIST_CMD | BUFFERED_LIST;
#ifdef WITH_SSL
      }
#endif

      if ((status = ftp_list(db.mode_flag, type, &list)) != SUCCESS)
      {
         trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, msg_str,
                   "Failed to send LIST command (%d).", status);
         (void)ftp_quit();
         exit(LIST_ERROR);
      }

      if (list != NULL)
      {
         /* Get all file masks for this directory. */
         if ((j = read_file_mask(fra[db.fra_pos].dir_alias, &nfg, &fml)) == INCORRECT)
         {
            if (j == LOCKFILE_NOT_THERE)
            {
               system_log(ERROR_SIGN, __FILE__, __LINE__,
                          "Failed to set lock in file masks for %s, because the file is not there.",
                          fra[db.fra_pos].dir_alias);
            }
            else if (j == LOCK_IS_SET)
                 {
                    system_log(ERROR_SIGN, __FILE__, __LINE__,
                               "Failed to get the file masks for %s, because lock is already set",
                               fra[db.fra_pos].dir_alias);
                 }
                 else
                 {
                    system_log(ERROR_SIGN, __FILE__, __LINE__,
                               "Failed to get the file masks for %s. (%d)",
                               fra[db.fra_pos].dir_alias, j);
                 }
            if (fml != NULL)
            {
               free(fml);
            }
            (void)ftp_quit();
            exit(INCORRECT);
         }

#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
         if ((fra[db.fra_pos].stupid_mode == YES) ||
             (fra[db.fra_pos].remove == YES))
         {
            if (reset_ls_data(db.fra_pos) == INCORRECT)
            {
               (void)ftp_quit();
               exit(INCORRECT);
            }
         }
         else
         {
            if (rl_fd == -1)
            {
               if (attach_ls_data(db.fra_pos, db.fsa_pos, db.special_flag,
                                  YES) == INCORRECT)
               {
                  (void)ftp_quit();
                  exit(INCORRECT);
               }
            }
         }
#else
         if (rl_fd == -1)
         {
            if (attach_ls_data(db.fra_pos, db.fsa_pos, db.special_flag,
                               YES) == INCORRECT)
            {
               (void)ftp_quit();
               exit(INCORRECT);
            }
         }
         if ((fra[db.fra_pos].stupid_mode == YES) ||
             (fra[db.fra_pos].remove == YES))
         {
            /*
             * If all files from the previous listing have been
             * collected, lets reset the ls_data structure or otherwise
             * it keeps on growing forever.
             */
# ifdef LOCK_DEBUG
            if (lock_region(rl_fd, LOCK_RETR_PROC,
                            __FILE__, __LINE__) == LOCK_IS_NOT_SET)
# else
            if (lock_region(rl_fd, LOCK_RETR_PROC) == LOCK_IS_NOT_SET)
# endif
            {
               if (reset_ls_data(db.fra_pos) == INCORRECT)
               {
                  (void)ftp_quit();
                  exit(INCORRECT);
               }
            }
# ifdef LOCK_DEBUG
            unlock_region(rl_fd, LOCK_RETR_PROC, __FILE__, __LINE__);
# else
            unlock_region(rl_fd, LOCK_RETR_PROC);
# endif
         }
#endif

         if ((fra[db.fra_pos].ignore_file_time != 0) ||
             (fra[db.fra_pos].delete_files_flag & UNKNOWN_FILES))
         {
            /* Note: FTP returns GMT so we need to convert this to GMT! */
            current_time = time(NULL);
            p_tm = gmtime(&current_time);
            current_time = mktime(p_tm);
         }

         /*
          * Evaluate the list from the LIST command.
          */
         p_end = list;
         do
         {
            p_start = p_end;
            while ((*p_end != '\r') && (*p_end != '\n') && (*p_end != '\0'))
            {
               p_end++;
            }

            if ((ftpparse(&fp, &file_size, &file_mtime, p_start, p_end - p_start) == 1) &&
                ((fp.flagtryretr == 1) &&
                 ((fp.name[0] != '.') ||
                  (fra[db.fra_pos].dir_flag & ACCEPT_DOT_FILES))))
            {
               list_length++;
               list_size += file_size;

               if (fp.namelen < MAX_FILENAME_LENGTH)
               {
                  /* Store file name */
                  (void)memcpy(file_name, fp.name, fp.namelen);
                  file_name[fp.namelen] = '\0';

                  if (fra[db.fra_pos].dir_flag == ALL_DISABLED)
                  {
                     delete_remote_file(FTP, file_name, fp.namelen,
#ifdef _DELETE_LOG
                                        DELETE_HOST_DISABLED,
#endif
                                        &files_deleted,
                                        &file_size_deleted, file_size);
                  }
                  else
                  {
                     gotcha = NO;
                     for (k = 0; k < nfg; k++)
                     {
                        p_mask = fml[k].file_list;
                        for (j = 0; j < fml[k].fc; j++)
                        {
                           if ((status = pmatch(p_mask, file_name, NULL)) == 0)
                           {
                              if (check_list(file_name, file_size, file_mtime,
                                             &files_to_retrieve,
                                             file_size_to_retrieve,
                                             more_files_in_list) == 0)
                              {
                                 gotcha = YES;
                              }
                              else
                              {
                                 gotcha = NEITHER;
                              }
                              break;
                           }
                           else if (status == 1)
                                {
                                   /* This file is definitly NOT wanted! */
                                   /* Lets skip the rest of this group.  */
                                   break;
                                }
#ifdef SHOW_FILTER_MISSES
                           if ((status == -1) ||
                               (fsa->debug > NORMAL_MODE))
                           {
                              char tmp_mask[MAX_FILENAME_LENGTH];

                              if (expand_filter(p_mask, tmp_mask, time(NULL)) == YES)
                              {
                                 trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL,
                                              "%s (%s) not fitting %s",
                                              p_mask, tmp_mask, file_name);
                              }
                              else
                              {
                                 trans_db_log(INFO_SIGN, __FILE__, __LINE__, NULL,
                                              "%s not fitting %s",
                                              p_mask, file_name);
                              }
                           }
#endif
                           NEXT(p_mask);
                        }
                        if ((gotcha == YES) || (gotcha == NEITHER))
                        {
                           break;
                        }
                     }

                     if ((gotcha == NO) && (status != 0) &&
                         (fra[db.fra_pos].delete_files_flag & UNKNOWN_FILES))
                     {
                        time_t diff_time = current_time - file_mtime;

                        if ((fra[db.fra_pos].unknown_file_time == -2) ||
                            ((diff_time > fra[db.fra_pos].unknown_file_time) &&
                             (diff_time > DEFAULT_TRANSFER_TIMEOUT)))
                        {
                           delete_remote_file(FTP, file_name, fp.namelen,
#ifdef _DELETE_LOG
                                              DEL_UNKNOWN_FILE,
#endif
                                              &files_deleted,
                                              &file_size_deleted, file_size);
                        }
                     }
                  }
               }
               else
               {
                  (void)memcpy(file_name, fp.name, MAX_FILENAME_LENGTH);
                  file_name[MAX_FILENAME_LENGTH] = '\0';
                  trans_log(ERROR_SIGN, __FILE__, __LINE__, NULL, NULL,
                            "Remote file name `%s' is to long, it may only be %d bytes long.",
                            file_name, MAX_FILENAME_LENGTH);
               }
            }
            while ((*p_end == '\r') || (*p_end == '\n'))
            {
               p_end++;
            }
         } while (*p_end != '\0');

         free(list);

         /* Free file mask list. */
         for (i = 0; i < nfg; i++)
         {
            free(fml[i].file_list);
         }
         free(fml);
      }

      if (files_deleted > 0)
      {
         trans_log(DEBUG_SIGN, NULL, 0, NULL, NULL,
#if SIZEOF_OFF_T == 4
                   "%d files %ld bytes found for retrieving %s[%u files with %ld bytes in %s (deleted %u files with %ld bytes)]. @%x",
#else
                   "%d files %lld bytes found for retrieving %s[%u files with %lld bytes in %s (deleted %u files with %lld bytes)]. @%x",
#endif
                   files_to_retrieve, (pri_off_t)(*file_size_to_retrieve),
                   (*more_files_in_list == YES) ? "(+) " : "",
                   list_length, (pri_off_t)list_size,
                   (db.target_dir[0] == '\0') ? "home dir" : db.target_dir,
                   files_deleted, (pri_off_t)file_size_deleted, db.id.dir);
      }
      else
      {
         trans_log(DEBUG_SIGN, NULL, 0, NULL, NULL,
#if SIZEOF_OFF_T == 4
                   "%d files %ld bytes found for retrieving %s[%u files with %ld bytes in %s]. @%x",
#else
                   "%d files %lld bytes found for retrieving %s[%u files with %lld bytes in %s]. @%x",
#endif
                   files_to_retrieve, (pri_off_t)(*file_size_to_retrieve),
                   (*more_files_in_list == YES) ? "(+) " : "",
                   list_length, (pri_off_t)list_size,
                   (db.target_dir[0] == '\0') ? "home dir" : db.target_dir,
                   db.id.dir);
      }

      /*
       * Remove all files from the remote_list structure that are not
       * in the current buffer.
       */
      if ((fra[db.fra_pos].stupid_mode != YES) &&
          (fra[db.fra_pos].remove == NO))
      {
         int    files_removed = 0,
                i;
         size_t move_size;

         for (i = 0; i < (no_of_listed_files - files_removed); i++)
         {
            if (rl[i].in_list == NO)
            {
               int j = i;

               while ((rl[j].in_list == NO) &&
                      (j < (no_of_listed_files - files_removed)))
               {
                  j++;
               }
               if (j != (no_of_listed_files - files_removed))
               {
                  move_size = (no_of_listed_files - files_removed - j) *
                              sizeof(struct retrieve_list);
                  (void)memmove(&rl[i], &rl[j], move_size);
               }
               files_removed += (j - i);
            }
         }

         if (files_removed > 0)
         {
            int    current_no_of_listed_files = no_of_listed_files;
            size_t new_size,
                   old_size;

            no_of_listed_files -= files_removed;
            if (no_of_listed_files < 0)
            {
               system_log(DEBUG_SIGN, __FILE__, __LINE__,
                          "Hmmm, no_of_listed_files = %d", no_of_listed_files);
               no_of_listed_files = 0;
            }
            if (no_of_listed_files == 0)
            {
               new_size = (RETRIEVE_LIST_STEP_SIZE * sizeof(struct retrieve_list)) +
                          AFD_WORD_OFFSET;
            }
            else
            {
               new_size = (((no_of_listed_files / RETRIEVE_LIST_STEP_SIZE) + 1) *
                           RETRIEVE_LIST_STEP_SIZE * sizeof(struct retrieve_list)) +
                          AFD_WORD_OFFSET;
            }
            old_size = (((current_no_of_listed_files / RETRIEVE_LIST_STEP_SIZE) + 1) *
                        RETRIEVE_LIST_STEP_SIZE * sizeof(struct retrieve_list)) +
                       AFD_WORD_OFFSET;

            if (old_size != new_size)
            {
               char *ptr;

               ptr = (char *)rl - AFD_WORD_OFFSET;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
               if ((fra[db.fra_pos].stupid_mode == YES) ||
                   (fra[db.fra_pos].remove == YES))
               {
                  if ((ptr = realloc(ptr, new_size)) == NULL)
                  {
                     system_log(ERROR_SIGN, __FILE__, __LINE__,
                                "realloc() error : %s", strerror(errno));
                     (void)ftp_quit();
                     exit(INCORRECT);
                  }
               }
               else
               {
#endif
                  if ((ptr = mmap_resize(rl_fd, ptr, new_size)) == (caddr_t) -1)
                  {
                     system_log(ERROR_SIGN, __FILE__, __LINE__,
                                "mmap_resize() error : %s", strerror(errno));
                     (void)ftp_quit();
                     exit(INCORRECT);
                  }
                  rl_size = new_size;
#ifdef DO_NOT_PARALLELIZE_ALL_FETCH
               }
#endif
               ptr += AFD_WORD_OFFSET;
               rl = (struct retrieve_list *)ptr;
            }
            *(int *)((char *)rl - AFD_WORD_OFFSET) = no_of_listed_files;
         }
      }
   }

   return(files_to_retrieve);
}
Example #4
0
/*########################## convert_ls_data() ##########################*/
char *
convert_ls_data(int           old_rl_fd,
                char          *old_rl_file,
                off_t         *old_rl_size,
                int           old_no_of_listed_files,
                char          *old_rl_ptr,
                unsigned char old_version,
                unsigned char new_version)
{
   char *ptr;

   if ((old_version == 0) && (new_version == 1))
   {
      int                    i;
      size_t                 new_size;
      struct retrieve_list_0 *old_rl;
      struct retrieve_list_1 *new_rl;

      old_rl = (struct retrieve_list_0 *)old_rl_ptr;
      new_size = old_no_of_listed_files * sizeof(struct retrieve_list_1);
      if ((ptr = malloc(new_size)) == NULL)
      {
         system_log(ERROR_SIGN, __FILE__, __LINE__,
                    "malloc() error [%d %d] : %s",
                    old_no_of_listed_files, new_size, strerror(errno));
         ptr = (char *)old_rl;
         ptr -= AFD_WORD_OFFSET_0;
#ifdef HAVE_MMAP
         if (munmap(ptr, *old_rl_size) == -1)
#else
         if (munmap_emu(ptr) == -1)
#endif
         {
            system_log(WARN_SIGN, __FILE__, __LINE__,
                       "Failed to munmap() %s : %s",
                       old_rl_file, strerror(errno));
         }
         *old_rl_size = -1;
         return(NULL);
      }
      (void)memset(ptr, 0, new_size);
      new_rl = (struct retrieve_list_1 *)ptr;

      /*
       * Copy all the old data into the new region.
       */
      for (i = 0; i < old_no_of_listed_files; i++)
      {
         (void)strcpy(new_rl[i].file_name, old_rl[i].file_name);
         new_rl[i].assigned     = 0;
         new_rl[i].special_flag = 0;
         new_rl[i].got_date     = old_rl[i].got_date;
         new_rl[i].retrieved    = old_rl[i].retrieved;
         new_rl[i].in_list      = old_rl[i].in_list;
         new_rl[i].size         = old_rl[i].size;
         new_rl[i].file_mtime   = old_rl[i].file_mtime;
      }

      ptr = (char *)old_rl;
      ptr -= AFD_WORD_OFFSET_0;

      /*
       * Resize the old retrieve list to the size of new one and then copy
       * the new structure into it. Then update the retrieve list version
       * number.
       */
      if ((ptr = mmap_resize(old_rl_fd, ptr, new_size + AFD_WORD_OFFSET_1)) == (caddr_t) -1)
      {
         system_log(ERROR_SIGN, __FILE__, __LINE__,
                    "Failed to mmap_resize() %s : %s",
                    old_rl_file, strerror(errno));
         free((void *)new_rl);
         return(NULL);
      }

      ptr += AFD_WORD_OFFSET_1;
      (void)memcpy(ptr, new_rl, new_size);
      free((void *)new_rl);    
      ptr -= AFD_WORD_OFFSET_1; 
      *(ptr + SIZEOF_INT + 1 + 1) = 0;               /* Not used. */
      *(ptr + SIZEOF_INT + 1 + 1 + 1) = new_version;
      *(int *)(ptr + SIZEOF_INT + 4) = 0;            /* Not used. */
      *(ptr + SIZEOF_INT + 4 + SIZEOF_INT) = 0;      /* Not used. */
      *(ptr + SIZEOF_INT + 4 + SIZEOF_INT + 1) = 0;  /* Not used. */
      *(ptr + SIZEOF_INT + 4 + SIZEOF_INT + 2) = 0;  /* Not used. */
      *(ptr + SIZEOF_INT + 4 + SIZEOF_INT + 3) = 0;  /* Not used. */
      *old_rl_size = new_size + AFD_WORD_OFFSET_1;

      system_log(INFO_SIGN, NULL, 0,
                 "Converted retrieve list for %s from version %d to %d.",
                 basename(old_rl_file), (int)old_version, (int)new_version);
   }
   else
   {
      system_log(ERROR_SIGN, NULL, 0,
                 "Don't know how to convert a version %d of AFD ls data type to version %d.",
                 old_version, new_version);
      ptr = NULL;
   }

   return(ptr);
}