void from_recordset_stream(const std::string &reliable_storage_local_root,
const std::string &reliable_storage_remote_root,
Input_Stream &input, Output_Stream &output,
const std::vector<rel::rlang::token> &tokens) {
/* Get the headers. */
record headings(input.parse_headings());
// Get the resource id field id, this is the only field that we're interested
// in, the rest are just placeholders.
size_t resource_id_field_id = headings.mandatory_find_field(
NP1_REL_DISTRIBUTED_RESOURCE_ID_FIELD_NAME);
// Create the target recordset.
std::string target_recordset_name(rel::rlang::compiler::eval_to_string_only(tokens));
np1::io::reliable_storage::id target_recordset_resource_id(target_recordset_name);
np1::io::reliable_storage rs(reliable_storage_local_root, reliable_storage_remote_root);
np1::io::reliable_storage::stream target_recordset_stream(rs);
NP1_ASSERT(rs.create_wo(target_recordset_resource_id, target_recordset_stream),
"Unable to create recordset " + target_recordset_name);
// Read all the input recordset chunks ids and write them to the recordset file.
input.parse_records(
shallow_copy_chunk_record_callback(
rs, target_recordset_stream, resource_id_field_id, headings.ref()));
NP1_ASSERT(target_recordset_stream.close(),
"Unable to close target recordset stream");
}
void from_data_stream(const std::string &reliable_storage_local_root,
const std::string &reliable_storage_remote_root,
Input_Stream &input, Output_Stream &output,
const std::vector<rel::rlang::token> &tokens) {
/* Get the headers. */
record headings(input.parse_headings());
// Interpret the arguments.
std::vector<std::pair<std::string, rlang::dt::data_type> > args = rel::rlang::compiler::eval_to_strings(tokens);
NP1_ASSERT((args.size() > 0) && (args.size() <= 2), "Incorrect number of arguments to rel.recordset.create");
tokens[0].assert(rlang::dt::data_type::TYPE_STRING == args[0].second,
"First argument to rel.recordset.create is not a string");
std::string target_recordset_name = args[0].first;
uint64_t approx_max_chunk_size = DEFAULT_APPROX_MAX_CHUNK_SIZE_BYTES;
if (args.size() > 1) {
tokens[0].assert((rlang::dt::data_type::TYPE_INT == args[1].second)
|| (rlang::dt::data_type::TYPE_UINT == args[1].second),
"Second argument to rel.recordset.create is not an integer");
approx_max_chunk_size = str::dec_to_int64(args[1].first);
}
// Get the stream that will hold the recordset.
np1::io::reliable_storage::id target_recordset_resource_id(target_recordset_name);
np1::io::reliable_storage rs(reliable_storage_local_root, reliable_storage_remote_root);
np1::io::reliable_storage::stream target_recordset_stream(rs);
NP1_ASSERT(rs.create_wo(target_recordset_resource_id, target_recordset_stream),
"Unable to create recordset " + target_recordset_name
+ " in reliable storage '" + reliable_storage_local_root + "'");
np1::io::reliable_storage::stream current_target_chunk_stream(rs);
buffered_reliable_storage_stream_type buffered_current_target_chunk_stream(
current_target_chunk_stream);
mandatory_reliable_storage_stream_type mandatory_current_target_chunk_stream(
buffered_current_target_chunk_stream);
np1::io::reliable_storage::id current_target_chunk_id;
uint64_t current_target_chunk_size = 0;
// Read all the input data and redistribute it into recordset chunks.
input.parse_records(
chunk_data_record_callback(
rs, target_recordset_stream, current_target_chunk_id,
current_target_chunk_stream, mandatory_current_target_chunk_stream,
current_target_chunk_size, approx_max_chunk_size, headings.ref()));
// Close everything.
mandatory_current_target_chunk_stream.close();
NP1_ASSERT(target_recordset_stream.close(),
"Unable to close target recordset stream");
}
void operator()(Input_Stream &input, Output_Stream &output,
const std::vector<rel::rlang::token> &tokens,
sort_type_type sort_type,
sort_order_type sort_order) {
NP1_ASSERT(tokens.size() > 0, "Unexpected empty stream operator argument list");
// Read the first line of input, we need it to add meaning to the arguments.
record headings(input.parse_headings());
std::vector<std::string> arg_headings;
rlang::compiler::compile_heading_name_list(
tokens, headings.ref(), arg_headings);
// Create the compare specs.
detail::compare_specs comp_specs(headings, arg_headings);
// Write out the headings then do the actual sorting.
headings.write(output);
less_than lt(comp_specs);
greater_than gt(comp_specs);
switch (sort_type) {
case TYPE_MERGE_SORT:
switch (sort_order) {
case ORDER_ASCENDING:
sort<detail::merge_sort>(input, output, lt);
break;
case ORDER_DESCENDING:
sort<detail::merge_sort>(input, output, gt);
break;
}
break;
case TYPE_QUICK_SORT:
//TODO: why the dickens isn't this quick sort and why is quick sort broken?
switch (sort_order) {
case ORDER_ASCENDING:
sort<detail::merge_sort>(input, output, lt);
break;
case ORDER_DESCENDING:
sort<detail::merge_sort>(input, output, gt);
break;
}
break;
}
}
list_base & emitHeadings( string pipedelimited )
{
ocString headings(pipedelimited);
string list_begin = listTemplate.getParagraph("list_begin");
string tr = listTemplate.getParagraph("tr");
ocString th = listTemplate.getParagraph("th");
string end_tr = listTemplate.getParagraph("end_tr");
webIO << list_begin << tr;
while( ! headings.endOfParse() )
{
colCount++;
string heading = headings.parse("|");
webIO << th.replace( "$data$", heading.c_str() );
}
webIO << end_tr;
return * this;
}
void operator()() {
// Rewind the child output file and set up the stream we'll use to read it.
io::file child_output_file;
rewind(m_child_output_fp);
child_output_file.from_handle(fileno(m_child_output_fp));
typedef io::mandatory_record_input_stream<io::file, rel::record, rel::record_ref> child_output_stream_type;
child_output_stream_type child_output_stream(child_output_file);
// Read the headings, writing them only if they haven't already been written.
rel::record headings(child_output_stream.parse_headings());
if (!m_output_headings_written) {
headings.write(m_final_output);
m_output_headings_written = true;
}
// Write out the stream body.
child_output_stream.copy(m_final_output);
// Clean up.
fclose(m_child_output_fp);
}
void operator()(const std::string &reliable_storage_local_root,
const std::string &reliable_storage_remote_root,
const std::string &listen_endpoint,
Input_Stream &input, Output_Stream &output,
const std::vector<rel::rlang::token> &tokens) {
log_info("Reading headers and compiling expression against headers.");
/* Get the headers. */
record headings(input.parse_headings());
// Compile, just to check that the headings & tokens are likely to work
// when we distribute.
record empty_headings;
rlang::vm vm = rlang::compiler::compile_single_expression(
tokens, headings.ref(), empty_headings.ref());
// Check that the expression is actually a boolean expression.
NP1_ASSERT(vm.return_type() == rlang::dt::TYPE_BOOL,
"Expression is not a boolean expression");
// Do the distribution.
distributed::distribute(log_id(), headings, headings, "rel.where", reliable_storage_local_root,
reliable_storage_remote_root, listen_endpoint, input, output, tokens);
}
omegaMapBase& omegaMapBase::headings (ofstream& out) {
return headings (out, tab);
}
void inputdata2(pointptr2 treenode)
{
/* input the names and character state data for species */
/* used in Mix & Penny */
long i, j, l;
char k;
Char charstate;
/* possible states are '0', '1', 'P', 'B', and '?' */
if (printdata)
headings(chars, "Characters", "----------");
for (i = 0; i < (chars); i++)
extras[i] = 0;
for (i = 1; i <= spp; i++) {
initname(i-1);
if (printdata) {
for (j = 0; j < nmlngth; j++)
putc(nayme[i - 1][j], outfile);
}
fprintf(outfile, " ");
for (j = 0; j < (words); j++) {
treenode[i - 1]->fulstte1[j] = 0;
treenode[i - 1]->fulstte0[j] = 0;
treenode[i - 1]->empstte1[j] = 0;
treenode[i - 1]->empstte0[j] = 0;
}
for (j = 1; j <= (chars); j++) {
k = (j - 1) % bits + 1;
l = (j - 1) / bits + 1;
do {
if (eoln(infile))
scan_eoln(infile);
charstate = gettc(infile);
if (charstate == '\n' || charstate == '\t')
charstate = ' ';
} while (charstate == ' ');
if (charstate == 'b') charstate = 'B';
if (charstate == 'p') charstate = 'P';
if (charstate != '0' && charstate != '1' && charstate != '?' &&
charstate != 'P' && charstate != 'B') {
printf("\n\nERROR: Bad character state: %c ",charstate);
printf("at character %ld of species %ld\n\n", j, i);
exxit(-1);
}
if (printdata) {
newline(outfile, j, 55, nmlngth + 3);
putc(charstate, outfile);
if (j % 5 == 0)
putc(' ', outfile);
}
if (charstate == '1') {
treenode[i-1]->fulstte1[l-1] =
((long)treenode[i-1]->fulstte1[l-1]) | (1L << k);
treenode[i-1]->empstte1[l-1] =
treenode[i-1]->fulstte1[l-1];
}
if (charstate == '0') {
treenode[i-1]->fulstte0[l-1] =
((long)treenode[i-1]->fulstte0[l-1]) | (1L << k);
treenode[i-1]->empstte0[l-1] =
treenode[i-1]->fulstte0[l-1];
}
if (charstate == 'P' || charstate == 'B')
extras[j-1] += weight[j-1];
}
scan_eoln(infile);
if (printdata)
putc('\n', outfile);
}
fprintf(outfile, "\n\n");
} /* inputdata2 */
void inputdata(pointptr treenode,boolean dollo,boolean printdata,FILE *outfile)
{
/* input the names and character state data for species */
/* used in Dollop, Dolpenny, Dolmove, & Move */
long i, j, l;
char k;
Char charstate;
/* possible states are '0', '1', 'P', 'B', and '?' */
if (printdata)
headings(chars, "Characters", "----------");
for (i = 0; i < (chars); i++)
extras[i] = 0;
for (i = 1; i <= spp; i++) {
initname(i-1);
if (printdata) {
for (j = 0; j < nmlngth; j++)
putc(nayme[i - 1][j], outfile);
fprintf(outfile, " ");
}
for (j = 0; j < (words); j++) {
treenode[i - 1]->stateone[j] = 0;
treenode[i - 1]->statezero[j] = 0;
}
for (j = 1; j <= (chars); j++) {
k = (j - 1) % bits + 1;
l = (j - 1) / bits + 1;
do {
if (eoln(infile))
scan_eoln(infile);
charstate = gettc(infile);
} while (charstate == ' ' || charstate == '\t');
if (charstate == 'b')
charstate = 'B';
if (charstate == 'p')
charstate = 'P';
if (charstate != '0' && charstate != '1' && charstate != '?' &&
charstate != 'P' && charstate != 'B') {
printf("\n\nERROR: Bad character state: %c ",charstate);
printf("at character %ld of species %ld\n\n", j, i);
exxit(-1);
}
if (printdata) {
newline(outfile, j, 55, nmlngth + 3);
putc(charstate, outfile);
if (j % 5 == 0)
putc(' ', outfile);
}
if (charstate == '1')
treenode[i - 1]->stateone[l - 1] =
((long)treenode[i - 1]->stateone[l - 1]) | (1L << k);
if (charstate == '0')
treenode[i - 1]->statezero[l - 1] =
((long)treenode[i - 1]->statezero[l - 1]) | (1L << k);
if (charstate == 'P' || charstate == 'B') {
if (dollo)
extras[j - 1] += weight[j - 1];
else {
treenode[i - 1]->stateone[l - 1] =
((long)treenode[i - 1]->stateone[l - 1]) | (1L << k);
treenode[i - 1]->statezero[l - 1] =
((long)treenode[i - 1]->statezero[l - 1]) | (1L << k);
}
}
}
scan_eoln(infile);
if (printdata)
putc('\n', outfile);
}
if (printdata)
fprintf(outfile, "\n\n");
} /* inputdata */
