BOOL LASreaderMerged::read_point_default()
{
if (file_name_current == file_name_start)
{
if (!open_next_file()) return FALSE;
}
while (true)
{
if (lasreader->read_point())
{
point = lasreader->point;
p_count++;
return TRUE;
}
if (lasreaderbin)
{
header.number_of_points_by_return[0] += lasreader->header.number_of_points_by_return[0];
header.number_of_points_by_return[1] += lasreader->header.number_of_points_by_return[1];
header.number_of_points_by_return[2] += lasreader->header.number_of_points_by_return[2];
header.number_of_points_by_return[3] += lasreader->header.number_of_points_by_return[3];
header.number_of_points_by_return[4] += lasreader->header.number_of_points_by_return[4];
if (header.max_x < lasreader->header.max_x) header.max_x = lasreader->header.max_x;
if (header.max_y < lasreader->header.max_y) header.max_y = lasreader->header.max_y;
if (header.max_z < lasreader->header.max_z) header.max_z = lasreader->header.max_z;
if (header.min_x > lasreader->header.min_x) header.min_x = lasreader->header.min_x;
if (header.min_y > lasreader->header.min_y) header.min_y = lasreader->header.min_y;
if (header.min_z > lasreader->header.min_z) header.min_z = lasreader->header.min_z;
}
else if (lasreadertxt)
{
if (!populate_header)
{
header.number_of_point_records += lasreader->header.number_of_point_records;
header.number_of_points_by_return[0] += lasreader->header.number_of_points_by_return[0];
header.number_of_points_by_return[1] += lasreader->header.number_of_points_by_return[1];
header.number_of_points_by_return[2] += lasreader->header.number_of_points_by_return[2];
header.number_of_points_by_return[3] += lasreader->header.number_of_points_by_return[3];
header.number_of_points_by_return[4] += lasreader->header.number_of_points_by_return[4];
if (header.max_x < lasreader->header.max_x) header.max_x = lasreader->header.max_x;
if (header.max_y < lasreader->header.max_y) header.max_y = lasreader->header.max_y;
if (header.max_z < lasreader->header.max_z) header.max_z = lasreader->header.max_z;
if (header.min_x > lasreader->header.min_x) header.min_x = lasreader->header.min_x;
if (header.min_y > lasreader->header.min_y) header.min_y = lasreader->header.min_y;
if (header.min_z > lasreader->header.min_z) header.min_z = lasreader->header.min_z;
}
}
lasreader->close();
point.zero();
if (!open_next_file()) return FALSE;
}
return FALSE;
}
bool open_next_file(stream_status& st) {
// The stream must be released, with .reset(), before calling
// .next() on the streams_iterator_, to ensure that the
// streams_iterator_ noticed that we closed that stream before
// requesting a new one.
st.stream.reset();
st.stream = streams_iterator_.next();
if(!st.stream) {
st.type = DONE_TYPE;
return false;
}
++files_read_;
switch(st.stream->peek()) {
case EOF: return open_next_file(st);
case '>':
st.type = FASTA_TYPE;
ignore_line(*st.stream); // Pass header
++reads_read_;
break;
case '@':
st.type = FASTQ_TYPE;
ignore_line(*st.stream); // Pass header
++reads_read_;
break;
default:
throw std::runtime_error("Unsupported format"); // Better error management
}
return true;
}
stream_type next() {
locks::pthread::mutex_lock lock(mutex_);
stream_type res;
open_next_file(res);
if(!res)
open_next_pipe(res);
return res;
}
void open_next_file_wrapper(void *priv)
{
file_private_t *config = (file_private_t*) priv;
char *new_filename = create_next_filename(config);
if (!new_filename) {
VERBOSE(CL_ERROR, "OUTPUT FILE IFC[%d]: memory allocation failed.", config->ifc_idx);
} else {
open_next_file(config, new_filename);
}
}
bool joined_line_readers::getline(std::string& dst)
{
dst.clear();
while (dst.empty()) {
if (m_reader && m_reader->getline(dst)) {
m_current_line++;
break;
} else if (!open_next_file()) {
break;
}
}
return !dst.empty();
}
/// Size is the number of buffers to keep around. It should be
/// larger than the number of thread expected to read from this
/// class. nb_sequences is the number of sequences to read into a
/// buffer. 'begin' and 'end' are iterators to a range of istream.
whole_sequence_parser(uint32_t size, uint32_t nb_sequences,
uint32_t max_producers, StreamIterator& streams) :
super(max_producers, size),
streams_(max_producers),
streams_iterator_(streams)
{
for(auto it = super::element_begin(); it != super::element_end(); ++it) {
it->nb_filled = 0;
it->data.resize(nb_sequences);
}
for(uint32_t i = 0; i < max_producers; ++i) {
streams_.init(i);
open_next_file(streams_[i]);
}
}
init_io(int argc, char **argv, int up)
{
gargc = argc; gargv = argv;
output_file = stdout;
if (gargc <= 1) {
input_file = stdin;
title(up);
} else {
input_file = open_next_file();
}
if (input_file == (FILE *)0) {
ERROR("No input stream\n");
exit(1);
}
}
void sorted_dumper<storage_t,atomic_t>::_dump() {
std::ofstream _out;
assert(dump_mutex.try_lock());
if(one_file) {
_out.open(file_prefix.c_str());
} else {
open_next_file(file_prefix.c_str(), &file_index, _out);
}
out = &_out;
unique = distinct = total = max_count = 0;
tr.reset();
thread_info[0].writer.write_header(out);
exec_join(threads);
ary->zero_blocks(0, nb_blocks); // zero out last group of blocks
update_stats();
_out.close();
dump_mutex.unlock();
}
/// Max_producers is the maximum number of concurrent threads than
/// can produce data simultaneously. Size is the number of buffer to
/// keep around. It should be larger than the number of thread
/// expected to read from this class. buf_size is the size of each
/// buffer. A StreamIterator is expected to have a next() method,
/// which is thread safe, and which returns (move) a
/// std::unique<std::istream> object.
mer_overlap_sequence_parser(uint16_t mer_len, uint32_t max_producers, uint32_t size, size_t buf_size,
StreamIterator& streams) :
super(max_producers, size),
mer_len_(mer_len),
buf_size_(buf_size),
buffer(new char[size * buf_size]),
seam_buffer(new char[max_producers * (mer_len - 1)]),
streams_(max_producers),
streams_iterator_(streams),
files_read_(0), reads_read_(0)
{
for(sequence_ptr* it = super::element_begin(); it != super::element_end(); ++it)
it->start = it->end = buffer + (it - super::element_begin()) * buf_size;
for(uint32_t i = 0; i < max_producers; ++i) {
streams_.init(i);
streams_[i].seam = seam_buffer + i * (mer_len - 1);
open_next_file(streams_[i]);
}
}
inline bool produce(uint32_t i, sequence_list& buff) {
stream_status& st = streams_[i];
switch(st.type) {
case FASTA_TYPE:
read_fasta(st, buff);
break;
case FASTQ_TYPE:
read_fastq(st, buff);
break;
case DONE_TYPE:
return true;
}
if(st.stream->good())
return false;
// Reach the end of file, close current and try to open the next one
open_next_file(st);
return false;
}
void open_next_file(stream_status& st) {
st.stream.reset();
st.stream = streams_iterator_.next();
if(!st.stream) {
st.type = DONE_TYPE;
return;
}
// Update the type of the current file and move past first header
// to beginning of sequence.
switch(st.stream->peek()) {
case EOF: return open_next_file(st);
case '>':
st.type = FASTA_TYPE;
break;
case '@':
st.type = FASTQ_TYPE;
break;
default:
throw std::runtime_error("Unsupported format"); // Better error management
}
}
/**
* \brief Write data to a file.
* Data to write are expected as a trap_buffer_header_t structure, thus actual length of data to be written is determined from trap_buffer_header_t->data_length
* trap_buffer_header_t->data_length is expected to be in network byte order (little endian)
*
* \param[in] priv pointer to module private data
* \param[in] data pointer to data to write
* \param[in] size size of data to write - NOT USED IN THIS INTERFACE
* \param[in] timeout NOT USED IN THIS INTERFACE
* \return 0 on success (TRAP_E_OK), TTRAP_E_IO_ERROR if error occurs during writing, TRAP_E_TERMINATED if interface was terminated.
*/
int file_send(void *priv, const void *data, uint32_t size, int timeout)
{
int ret_val = 0;
file_private_t *config = (file_private_t*) priv;
size_t written;
if (config->is_terminated) {
return trap_error(config->ctx, TRAP_E_TERMINATED);
}
/* Check whether the file stream is opened */
if (config->fd == NULL) {
return trap_error(config->ctx, TRAP_E_NOT_INITIALIZED);
}
#ifdef ENABLE_NEGOTIATION
if (config->neg_initialized == 0) {
ret_val = output_ifc_negotiation((void *) config, TRAP_IFC_TYPE_FILE, 0);
if (ret_val == NEG_RES_OK) {
VERBOSE(CL_VERBOSE_LIBRARY, "File output_ifc_negotiation result: success.");
config->neg_initialized = 1;
fflush(config->fd);
} else if (ret_val == NEG_RES_FMT_UNKNOWN) {
VERBOSE(CL_VERBOSE_LIBRARY, "File output_ifc_negotiation result: failed (unknown data format of this output interface -> refuse client).");
return trap_error(config->ctx, TRAP_E_NOT_INITIALIZED);
} else { /* ret_val == NEG_RES_FAILED */
VERBOSE(CL_VERBOSE_LIBRARY, "File output_ifc_negotiation result: failed (error while sending hello message to input interface).");
return trap_error(config->ctx, TRAP_E_NOT_INITIALIZED);
}
}
#endif
/* Writes data_length bytes to the file */
written = fwrite(data, 1, size, config->fd);
if (written != size) {
VERBOSE(CL_ERROR, "OUTPUT FILE IFC: unable to write to file: %s", config->filename);
return trap_errorf(config->ctx, TRAP_E_IO_ERROR, "OUTPUT FILE IFC: unable to write");
}
if (config->file_change_time != 0) {
time_t current_time = time(NULL);
if (difftime(current_time, config->starting_time) / 60 >= config->file_change_time) {
char *new_filename = create_filename_from_time(priv);
if (!new_filename) {
VERBOSE(CL_ERROR, "OUTPUT FILE IFC[%d]: memory allocation failed.", config->ifc_idx);
return trap_error(config->ctx, TRAP_E_MEMORY);
}
if (open_next_file(priv, new_filename) < 0) {
VERBOSE(CL_ERROR, "OUTPUT FILE IFC[%d]: opening new file failed.", config->ifc_idx);
return trap_errorf(config->ctx, TRAP_E_BADPARAMS, "unable to open file");
}
}
}
if (config->file_change_size != 0 && (uint64_t)ftell(config->fd) >= (uint64_t)(1024 * 1024 * (uint64_t)config->file_change_size)) {
char *new_filename = create_next_filename(priv);
if (!new_filename) {
VERBOSE(CL_ERROR, "OUTPUT FILE IFC[%d]: memory allocation failed.", config->ifc_idx);
return trap_error(config->ctx, TRAP_E_MEMORY);
}
if (open_next_file(priv, new_filename) < 0) {
VERBOSE(CL_ERROR, "OUTPUT FILE IFC[%d]: opening new file failed.", config->ifc_idx);
return trap_errorf(config->ctx, TRAP_E_BADPARAMS, "unable to open file");
}
}
return TRAP_E_OK;
}
/**
* \brief Read data from a file.
* \param[in] priv pointer to module private data
* \param[out] data pointer to a memory block in which data is to be stored
* \param[out] size pointer to a memory block in which size of read data is to be stored
* \param[in] timeout NOT USED IN THIS INTERFACE
* \return 0 on success (TRAP_E_OK), TRAP_E_IO_ERROR if error occurs during reading, TRAP_E_TERMINATED if interface was terminated.
*/
int file_recv(void *priv, void *data, uint32_t *size, int timeout)
{
size_t loaded;
char *next_file = NULL;
/* header of message inside the buffer */
uint16_t *m_head = data;
uint32_t data_size = 0;
file_private_t *config = (file_private_t*) priv;
if (config->is_terminated) {
return trap_error(config->ctx, TRAP_E_TERMINATED);
}
/* Check whether the file stream is opened */
if (config->fd == NULL) {
return trap_error(config->ctx, TRAP_E_NOT_INITIALIZED);
}
#ifdef ENABLE_NEGOTIATION
neg_start:
if (config->neg_initialized == 0) {
switch(input_ifc_negotiation((void *) config, TRAP_IFC_TYPE_FILE)) {
case NEG_RES_FMT_UNKNOWN:
VERBOSE(CL_VERBOSE_LIBRARY, "Input_ifc_negotiation result: failed (unknown data format of the output interface).");
return TRAP_E_FORMAT_MISMATCH;
case NEG_RES_CONT:
VERBOSE(CL_VERBOSE_LIBRARY, "Input_ifc_negotiation result: success.");
config->neg_initialized = 1;
break;
case NEG_RES_RECEIVER_FMT_SUBSET:
VERBOSE(CL_VERBOSE_LIBRARY, "Input_ifc_negotiation result: success (data specifier of the input interface is subset of the output interface data specifier).");
config->neg_initialized = 1;
break;
case NEG_RES_SENDER_FMT_SUBSET:
VERBOSE(CL_VERBOSE_LIBRARY, "Input_ifc_negotiation result: success (new data specifier of the output interface is subset of the old one; it was not first negotiation).");
config->neg_initialized = 1;
break;
case NEG_RES_FAILED:
VERBOSE(CL_VERBOSE_LIBRARY, "Input_ifc_negotiation result: failed (error while receiving hello message from output interface).");
return TRAP_E_FORMAT_MISMATCH;
case NEG_RES_FMT_MISMATCH:
VERBOSE(CL_VERBOSE_LIBRARY, "Input_ifc_negotiation result: failed (data format or data specifier mismatch).");
return TRAP_E_FORMAT_MISMATCH;
default:
VERBOSE(CL_VERBOSE_LIBRARY, "Input_ifc_negotiation result: default case");
break;
}
}
#endif
/* Reads 4 bytes from the file, determining the length of bytes to be read to @param[out] data */
loaded = fread(&data_size, sizeof(uint32_t), 1, config->fd);
if (loaded != 1) {
if (feof(config->fd)) {
next_file = get_next_file(priv);
if (!next_file) {
/* set size of buffer to the size of 1 message (including its header) */
(*size) = 2;
/* set the header of message to 0B */
*m_head = 0;
return TRAP_E_OK;
} else {
if (open_next_file(config, next_file) == 0) {
#ifdef ENABLE_NEGOTIATION
goto neg_start;
#endif
} else {
return trap_errorf(config->ctx, TRAP_E_IO_ERROR, "INPUT FILE IFC[%d]: unable to open next file.", config->ifc_idx);
}
}
} else {
VERBOSE(CL_ERROR, "INPUT FILE IFC: read error occurred in file: %s", config->filename);
return trap_errorf(config->ctx, TRAP_E_IO_ERROR, "INPUT FILE IFC: unable to read");
}
}
*size = ntohl(data_size);
/* Reads (*size) bytes from the file */
loaded = fread(data, 1, (*size), config->fd);
if (loaded != (*size)) {
VERBOSE(CL_ERROR, "INPUT FILE IFC: read incorrect number of bytes from file: %s. Attempted to read %d bytes, but the actual count of bytes read was %zu.", config->filename, (*size), loaded);
}
return TRAP_E_OK;
}