示例#1
0
BOOL LASreaderMerged::read_point_default()
{
  if (file_name_current == file_name_start)
  {
    if (!open_next_file()) return FALSE;


  }

  while (true)
  {
    if (lasreader->read_point())
    {
      point = lasreader->point;
      p_count++;
      return TRUE;
    }
    if (lasreaderbin)
    {
      header.number_of_points_by_return[0] += lasreader->header.number_of_points_by_return[0];
      header.number_of_points_by_return[1] += lasreader->header.number_of_points_by_return[1];
      header.number_of_points_by_return[2] += lasreader->header.number_of_points_by_return[2];
      header.number_of_points_by_return[3] += lasreader->header.number_of_points_by_return[3];
      header.number_of_points_by_return[4] += lasreader->header.number_of_points_by_return[4];
      if (header.max_x < lasreader->header.max_x) header.max_x = lasreader->header.max_x;
      if (header.max_y < lasreader->header.max_y) header.max_y = lasreader->header.max_y;
      if (header.max_z < lasreader->header.max_z) header.max_z = lasreader->header.max_z;
      if (header.min_x > lasreader->header.min_x) header.min_x = lasreader->header.min_x;
      if (header.min_y > lasreader->header.min_y) header.min_y = lasreader->header.min_y;
      if (header.min_z > lasreader->header.min_z) header.min_z = lasreader->header.min_z;
    }
    else if (lasreadertxt)
    {
      if (!populate_header)
      {
        header.number_of_point_records += lasreader->header.number_of_point_records;
        header.number_of_points_by_return[0] += lasreader->header.number_of_points_by_return[0];
        header.number_of_points_by_return[1] += lasreader->header.number_of_points_by_return[1];
        header.number_of_points_by_return[2] += lasreader->header.number_of_points_by_return[2];
        header.number_of_points_by_return[3] += lasreader->header.number_of_points_by_return[3];
        header.number_of_points_by_return[4] += lasreader->header.number_of_points_by_return[4];
        if (header.max_x < lasreader->header.max_x) header.max_x = lasreader->header.max_x;
        if (header.max_y < lasreader->header.max_y) header.max_y = lasreader->header.max_y;
        if (header.max_z < lasreader->header.max_z) header.max_z = lasreader->header.max_z;
        if (header.min_x > lasreader->header.min_x) header.min_x = lasreader->header.min_x;
        if (header.min_y > lasreader->header.min_y) header.min_y = lasreader->header.min_y;
        if (header.min_z > lasreader->header.min_z) header.min_z = lasreader->header.min_z;
      }
    }
    lasreader->close();
    point.zero();
    if (!open_next_file()) return FALSE;
  }
  return FALSE;
}
  bool open_next_file(stream_status& st) {
    // The stream must be released, with .reset(), before calling
    // .next() on the streams_iterator_, to ensure that the
    // streams_iterator_ noticed that we closed that stream before
    // requesting a new one.
    st.stream.reset();
    st.stream = streams_iterator_.next();
    if(!st.stream) {
      st.type = DONE_TYPE;
      return false;
    }

    ++files_read_;
    switch(st.stream->peek()) {
    case EOF: return open_next_file(st);
    case '>':
      st.type = FASTA_TYPE;
      ignore_line(*st.stream); // Pass header
      ++reads_read_;
      break;
    case '@':
      st.type = FASTQ_TYPE;
      ignore_line(*st.stream); // Pass header
      ++reads_read_;
      break;
    default:
      throw std::runtime_error("Unsupported format"); // Better error management
    }
    return true;
  }
示例#3
0
 stream_type next() {
   locks::pthread::mutex_lock lock(mutex_);
   stream_type res;
   open_next_file(res);
   if(!res)
     open_next_pipe(res);
   return res;
 }
示例#4
0
void open_next_file_wrapper(void *priv)
{
   file_private_t *config = (file_private_t*) priv;
   char *new_filename = create_next_filename(config);
   if (!new_filename) {
      VERBOSE(CL_ERROR, "OUTPUT FILE IFC[%d]: memory allocation failed.", config->ifc_idx);
   } else {
      open_next_file(config, new_filename);
   }
}
bool joined_line_readers::getline(std::string& dst)
{
    dst.clear();

    while (dst.empty()) {
        if (m_reader && m_reader->getline(dst)) {
            m_current_line++;
            break;
        } else if (!open_next_file()) {
            break;
        }
    }

    return !dst.empty();
}
 /// Size is the number of buffers to keep around. It should be
 /// larger than the number of thread expected to read from this
 /// class. nb_sequences is the number of sequences to read into a
 /// buffer. 'begin' and 'end' are iterators to a range of istream.
 whole_sequence_parser(uint32_t size, uint32_t nb_sequences,
                       uint32_t max_producers, StreamIterator& streams) :
   super(max_producers, size),
   streams_(max_producers),
   streams_iterator_(streams)
 {
   for(auto it = super::element_begin(); it != super::element_end(); ++it) {
     it->nb_filled = 0;
     it->data.resize(nb_sequences);
   }
   for(uint32_t i = 0; i < max_producers; ++i) {
     streams_.init(i);
     open_next_file(streams_[i]);
   }
 }
示例#7
0
init_io(int argc, char **argv, int up)
{
    gargc = argc; gargv = argv;

    output_file = stdout;

    if (gargc <= 1) {
        input_file = stdin;
        title(up);
    } else {
        input_file = open_next_file();
    }
    if (input_file == (FILE *)0) {
        ERROR("No input stream\n");
        exit(1);
    }
}
 void sorted_dumper<storage_t,atomic_t>::_dump() {
   std::ofstream _out;
   assert(dump_mutex.try_lock());
   if(one_file) {
     _out.open(file_prefix.c_str());
   } else {
     open_next_file(file_prefix.c_str(), &file_index, _out);
   }
   out = &_out;
   unique = distinct = total = max_count = 0;
   tr.reset();
   thread_info[0].writer.write_header(out);
   exec_join(threads);
   ary->zero_blocks(0, nb_blocks); // zero out last group of blocks
   update_stats();
   _out.close();
   dump_mutex.unlock();
 }
 /// Max_producers is the maximum number of concurrent threads than
 /// can produce data simultaneously. Size is the number of buffer to
 /// keep around. It should be larger than the number of thread
 /// expected to read from this class. buf_size is the size of each
 /// buffer. A StreamIterator is expected to have a next() method,
 /// which is thread safe, and which returns (move) a
 /// std::unique<std::istream> object.
 mer_overlap_sequence_parser(uint16_t mer_len, uint32_t max_producers, uint32_t size, size_t buf_size,
                             StreamIterator& streams) :
   super(max_producers, size),
   mer_len_(mer_len),
   buf_size_(buf_size),
   buffer(new char[size * buf_size]),
   seam_buffer(new char[max_producers * (mer_len - 1)]),
   streams_(max_producers),
   streams_iterator_(streams),
   files_read_(0), reads_read_(0)
 {
   for(sequence_ptr* it = super::element_begin(); it != super::element_end(); ++it)
     it->start = it->end = buffer + (it - super::element_begin()) * buf_size;
   for(uint32_t i = 0; i < max_producers; ++i) {
     streams_.init(i);
     streams_[i].seam = seam_buffer + i * (mer_len - 1);
     open_next_file(streams_[i]);
   }
 }
示例#10
0
  inline bool produce(uint32_t i, sequence_list& buff) {
    stream_status& st = streams_[i];

    switch(st.type) {
    case FASTA_TYPE:
      read_fasta(st, buff);
      break;
    case FASTQ_TYPE:
      read_fastq(st, buff);
      break;
    case DONE_TYPE:
      return true;
    }

    if(st.stream->good())
      return false;

    // Reach the end of file, close current and try to open the next one
    open_next_file(st);
    return false;
  }
  void open_next_file(stream_status& st) {
    st.stream.reset();
    st.stream = streams_iterator_.next();
    if(!st.stream) {
      st.type = DONE_TYPE;
      return;
    }

    // Update the type of the current file and move past first header
    // to beginning of sequence.
    switch(st.stream->peek()) {
    case EOF: return open_next_file(st);
    case '>':
      st.type = FASTA_TYPE;
      break;
    case '@':
      st.type = FASTQ_TYPE;
      break;
    default:
      throw std::runtime_error("Unsupported format"); // Better error management
    }
  }
示例#12
0
/**
 * \brief Write data to a file.
 * Data to write are expected as a trap_buffer_header_t structure, thus actual length of data to be written is determined from trap_buffer_header_t->data_length
 * trap_buffer_header_t->data_length is expected to be in network byte order (little endian)
 *
 * \param[in] priv   pointer to module private data
 * \param[in] data   pointer to data to write
 * \param[in] size   size of data to write - NOT USED IN THIS INTERFACE
 * \param[in] timeout   NOT USED IN THIS INTERFACE
 * \return 0 on success (TRAP_E_OK), TTRAP_E_IO_ERROR if error occurs during writing, TRAP_E_TERMINATED if interface was terminated.
 */
int file_send(void *priv, const void *data, uint32_t size, int timeout)
{
   int ret_val = 0;
   file_private_t *config = (file_private_t*) priv;
   size_t written;

   if (config->is_terminated) {
      return trap_error(config->ctx, TRAP_E_TERMINATED);
   }

   /* Check whether the file stream is opened */
   if (config->fd == NULL) {
      return trap_error(config->ctx, TRAP_E_NOT_INITIALIZED);
   }

#ifdef ENABLE_NEGOTIATION
   if (config->neg_initialized == 0) {
      ret_val = output_ifc_negotiation((void *) config, TRAP_IFC_TYPE_FILE, 0);
      if (ret_val == NEG_RES_OK) {
         VERBOSE(CL_VERBOSE_LIBRARY, "File output_ifc_negotiation result: success.");
         config->neg_initialized = 1;
         fflush(config->fd);
      } else if (ret_val == NEG_RES_FMT_UNKNOWN) {
         VERBOSE(CL_VERBOSE_LIBRARY, "File output_ifc_negotiation result: failed (unknown data format of this output interface -> refuse client).");
         return trap_error(config->ctx, TRAP_E_NOT_INITIALIZED);
      } else { /* ret_val == NEG_RES_FAILED */
         VERBOSE(CL_VERBOSE_LIBRARY, "File output_ifc_negotiation result: failed (error while sending hello message to input interface).");
         return trap_error(config->ctx, TRAP_E_NOT_INITIALIZED);
      }
   }
#endif

   /* Writes data_length bytes to the file */
   written = fwrite(data, 1, size, config->fd);
   if (written != size) {
      VERBOSE(CL_ERROR, "OUTPUT FILE IFC: unable to write to file: %s", config->filename);
      return trap_errorf(config->ctx, TRAP_E_IO_ERROR, "OUTPUT FILE IFC: unable to write");
   }

   if (config->file_change_time != 0) {
      time_t current_time = time(NULL);
      if (difftime(current_time, config->starting_time) / 60 >= config->file_change_time) {
         char *new_filename = create_filename_from_time(priv);
         if (!new_filename) {
            VERBOSE(CL_ERROR, "OUTPUT FILE IFC[%d]: memory allocation failed.", config->ifc_idx);
            return trap_error(config->ctx, TRAP_E_MEMORY);
         }

         if (open_next_file(priv, new_filename) < 0) {
            VERBOSE(CL_ERROR, "OUTPUT FILE IFC[%d]: opening new file failed.", config->ifc_idx);
            return trap_errorf(config->ctx, TRAP_E_BADPARAMS, "unable to open file");
         }
      }
   }

   if (config->file_change_size != 0 && (uint64_t)ftell(config->fd) >= (uint64_t)(1024 * 1024 * (uint64_t)config->file_change_size)) {
      char *new_filename = create_next_filename(priv);
      if (!new_filename) {
         VERBOSE(CL_ERROR, "OUTPUT FILE IFC[%d]: memory allocation failed.", config->ifc_idx);
         return trap_error(config->ctx, TRAP_E_MEMORY);
      }

         if (open_next_file(priv, new_filename) < 0) {
            VERBOSE(CL_ERROR, "OUTPUT FILE IFC[%d]: opening new file failed.", config->ifc_idx);
            return trap_errorf(config->ctx, TRAP_E_BADPARAMS, "unable to open file");
         }
   }

   return TRAP_E_OK;
}
示例#13
0
/**
 * \brief Read data from a file.
 * \param[in] priv   pointer to module private data
 * \param[out] data  pointer to a memory block in which data is to be stored
 * \param[out] size  pointer to a memory block in which size of read data is to be stored
 * \param[in] timeout   NOT USED IN THIS INTERFACE
 * \return 0 on success (TRAP_E_OK), TRAP_E_IO_ERROR if error occurs during reading, TRAP_E_TERMINATED if interface was terminated.
 */
int file_recv(void *priv, void *data, uint32_t *size, int timeout)
{
   size_t loaded;
   char *next_file = NULL;
   /* header of message inside the buffer */
   uint16_t *m_head = data;
   uint32_t data_size = 0;

   file_private_t *config = (file_private_t*) priv;

   if (config->is_terminated) {
      return trap_error(config->ctx, TRAP_E_TERMINATED);
   }

   /* Check whether the file stream is opened */
   if (config->fd == NULL) {
      return trap_error(config->ctx, TRAP_E_NOT_INITIALIZED);
   }

#ifdef ENABLE_NEGOTIATION
neg_start:
   if (config->neg_initialized == 0) {
      switch(input_ifc_negotiation((void *) config, TRAP_IFC_TYPE_FILE)) {
      case NEG_RES_FMT_UNKNOWN:
         VERBOSE(CL_VERBOSE_LIBRARY, "Input_ifc_negotiation result: failed (unknown data format of the output interface).");
         return TRAP_E_FORMAT_MISMATCH;

      case NEG_RES_CONT:
         VERBOSE(CL_VERBOSE_LIBRARY, "Input_ifc_negotiation result: success.");
         config->neg_initialized = 1;
         break;

      case NEG_RES_RECEIVER_FMT_SUBSET:
         VERBOSE(CL_VERBOSE_LIBRARY, "Input_ifc_negotiation result: success (data specifier of the input interface is subset of the output interface data specifier).");
         config->neg_initialized = 1;
         break;

      case NEG_RES_SENDER_FMT_SUBSET:
         VERBOSE(CL_VERBOSE_LIBRARY, "Input_ifc_negotiation result: success (new data specifier of the output interface is subset of the old one; it was not first negotiation).");
         config->neg_initialized = 1;
         break;

      case NEG_RES_FAILED:
         VERBOSE(CL_VERBOSE_LIBRARY, "Input_ifc_negotiation result: failed (error while receiving hello message from output interface).");
         return TRAP_E_FORMAT_MISMATCH;

      case NEG_RES_FMT_MISMATCH:
         VERBOSE(CL_VERBOSE_LIBRARY, "Input_ifc_negotiation result: failed (data format or data specifier mismatch).");
         return TRAP_E_FORMAT_MISMATCH;

      default:
         VERBOSE(CL_VERBOSE_LIBRARY, "Input_ifc_negotiation result: default case");
         break;
      }
   }
#endif

   /* Reads 4 bytes from the file, determining the length of bytes to be read to @param[out] data */
   loaded = fread(&data_size, sizeof(uint32_t), 1, config->fd);
   if (loaded != 1) {
      if (feof(config->fd)) {
         next_file = get_next_file(priv);
         if (!next_file) {
            /* set size of buffer to the size of 1 message (including its header) */
            (*size) = 2;
            /* set the header of message to 0B */
            *m_head = 0;

            return TRAP_E_OK;
         } else {
            if (open_next_file(config, next_file) == 0) {
#ifdef ENABLE_NEGOTIATION
               goto neg_start;
#endif
            } else {
               return trap_errorf(config->ctx, TRAP_E_IO_ERROR, "INPUT FILE IFC[%d]: unable to open next file.", config->ifc_idx);
            }
         }
      } else {
         VERBOSE(CL_ERROR, "INPUT FILE IFC: read error occurred in file: %s", config->filename);
         return trap_errorf(config->ctx, TRAP_E_IO_ERROR, "INPUT FILE IFC: unable to read");
      }
   }

   *size = ntohl(data_size);
   /* Reads (*size) bytes from the file */
   loaded = fread(data, 1, (*size), config->fd);
   if (loaded != (*size)) {
         VERBOSE(CL_ERROR, "INPUT FILE IFC: read incorrect number of bytes from file: %s. Attempted to read %d bytes, but the actual count of bytes read was %zu.", config->filename, (*size), loaded);
   }

   return TRAP_E_OK;
}