void query_streams(const char *path)
{
struct mfs_dirent *dir;
int parts[MAX_PARTS];
u32 count, i, j;
const char *ep;
dir = mfs_dir(mfs_resolve(path), &count);
for (i=0;i<count;i++) {
int n = query_int_list(dir[i].fsid, "Part/File", parts, MAX_PARTS );
fprintf(stdout, "%d\t", dir[i].fsid );
fprintf(stdout,"%s\t",
query_string(dir[i].fsid, "Showing/Program/Title"));
ep = query_string(dir[i].fsid, "Showing/Program/EpisodeTitle");
fprintf(stdout, "%s", ep?ep:"" );
for(j=0;j<n; j++)
fprintf(stdout, "\t%d", parts[j] );
fprintf(stdout,"\n");
}
if (dir) mfs_dir_free(dir);
}
/* get user defined size */
BOOL
gsview_usersize()
{
TCHAR prompt[MAXSTR];
char answer[MAXSTR];
nHelpTopic = IDS_TOPICMEDIA;
load_string(IDS_USERWIDTH, prompt, sizeof(prompt)/sizeof(TCHAR)-1);
put_points(answer, sizeof(answer), (float)option.user_width);
if (!query_string(prompt,answer))
return FALSE;
option.user_width = (int)(get_points(answer) + 0.5);
load_string(IDS_USERHEIGHT, prompt, sizeof(prompt)/sizeof(TCHAR)-1);
put_points(answer, sizeof(answer), (float)option.user_height);
if (!query_string(prompt,answer))
return FALSE;
option.user_height = (int)(get_points(answer) + 0.5);
if ((option.user_width==0) || (option.user_height == 0)) {
option.user_width = 595;
option.user_width = 842;
}
gsview_check_usersize();
return TRUE;
}
/*!
* \brief
* Splits a URL query string into individual field-value pairs.
*
* \param query
* Pointer to the query string to parse.
*
* Splits a URL query string into individual field-value pairs.
*/
void c_url_interface::ParseQuery(const char* query)
{
// field=value&field=&field=value
// remove any existing queries
ClearQueries();
if(!query)
return;
std::string query_string(query);
do
{
// split the query string at the next & if present to get the next query
std::string current_query;
SplitString(query_string, current_query, "&", true);
if(current_query.length() == 0)
{
// this is the last query so set the string to whatever is left
current_query = query_string;
query_string.clear();
}
// check the string contains an "="
if(std::string::npos == current_query.find("="))
{
// the query string is invalid, clear the query list and return
ClearQueries();
return;
}
std::string field;
std::string value;
// split at the "=" for the field and set the value to whatever is left
SplitString(current_query, field, "=", true);
field = Unescape(field, true);
value = Unescape(current_query, true);
AddQuery(field.c_str(), value.c_str());
}while(query_string.length());
}
std::map<std::string, std::string> atlas::http::detail::parse_get_parameters(
mg_connection *mg_conn
)
{
std::map<std::string, std::string> params;
if(mg_conn->query_string == nullptr)
return params;
std::string query_string(mg_conn->query_string);
typedef boost::tokenizer<boost::escaped_list_separator<char>> tokenizer;
tokenizer t(query_string, boost::escaped_list_separator<char>("\\", "&", "\""));
for(auto it = t.begin(); it != t.end(); ++it)
{
auto index = it->find('=');
if(index < 0) continue;
params[it->substr(0, index)] =
it->substr(index + 1);
}
return params;
}
wtss::cxx::geoarray_t wtss::cxx::client::describe_coverage(
const std::string& cv_name) const
{
std::string query_string("/describe_coverage?name=" + cv_name);
rapidjson::Document doc;
doc.Parse<0>(wtss::cxx::request(
wtss::cxx::client::server_uri + query_string).c_str());
if (doc.HasParseError())
{
boost::format err_msg("Error parsing requested document '%1%%2%': %3%.");
throw parse_error() << error_description(
(err_msg % server_uri % query_string %doc.GetParseError()).str());
}
if (!doc.IsObject())
throw parse_error() << error_description(
"Invalid JSON document: expecting a object!");
wtss::cxx::geoarray_t result;
if (!doc.HasMember("name"))
throw parse_error() << error_description(
"Invalid JSON document: expecting a member named \"coverage\"!");
result.name = doc["name"].GetString();
if (!doc.HasMember("description"))
throw parse_error() << error_description(
"Invalid JSON document: expecting a member named \"description\"!");
result.description = doc["description"].GetString();
if (!doc.HasMember("detail"))
throw parse_error() << error_description(
"Invalid JSON document: expecting a member named \"detail\"!");
result.detail = doc["detail"].GetString();
if (!doc.HasMember("dimensions"))
throw parse_error() << error_description(
"Invalid JSON document: expecting a member named \"dimensions\"!");
const rapidjson::Value& j_dimensions = doc["dimensions"];
if (!doc.HasMember("attributes"))
throw parse_error() << error_description(
"Invalid JSON document: expecting a member named \"attributes\"!");
const rapidjson::Value& j_attributes = doc["attributes"];
if (!j_dimensions.IsArray())
throw parse_error() << error_description(
"Invalid JSON document: member named \"dimensions\" must be an array!");
if (!j_attributes.IsArray())
throw parse_error() << error_description(
"Invalid JSON document: member named \"attributes\" must be an array!");
for (rapidjson::Value::ConstValueIterator itr = j_dimensions.Begin();
itr != j_dimensions.End(); ++itr)
{
wtss::cxx::dimension_t dimension;
dimension.name = (*itr)["name"].GetString();
dimension.description = (*itr)["description"].GetString();
dimension.max_idx = (*itr)["max_idx"].GetInt64();
dimension.min_idx = (*itr)["min_idx"].GetInt64();
result.dimensions.push_back(dimension);
}
for (rapidjson::Value::ConstValueIterator itr = j_attributes.Begin();
itr != j_attributes.End(); ++itr)
{
wtss::cxx::attribute_t attribute;
attribute.name = (*itr)["name"].GetString();
attribute.description = (*itr)["description"].GetString();
attribute.scale_factor = (*itr)["scale_factor"].GetDouble();
attribute.missing_value = (*itr)["missing_value"].GetDouble();
attribute.datatype =
wtss::cxx::datatype_t::from_string((*itr)["datatype"].GetString());
attribute.valid_range.max_val = (*itr)["valid_range"]["max"].GetDouble();
attribute.valid_range.min_val = (*itr)["valid_range"]["min"].GetDouble();
result.attributes.push_back(attribute);
}
const rapidjson::Value& j_geo_extent = doc["geo_extent"];
wtss::cxx::spatial_extent_t spatial_extent;
spatial_extent.extent.xmin =
j_geo_extent["spatial"]["extent"]["xmin"].GetDouble();
spatial_extent.extent.xmax =
j_geo_extent["spatial"]["extent"]["xmax"].GetDouble();
spatial_extent.extent.ymin =
j_geo_extent["spatial"]["extent"]["ymin"].GetDouble();
spatial_extent.extent.ymax =
j_geo_extent["spatial"]["extent"]["ymax"].GetDouble();
spatial_extent.resolution.x =
j_geo_extent["spatial"]["resolution"]["x"].GetDouble();
spatial_extent.resolution.y =
j_geo_extent["spatial"]["resolution"]["y"].GetDouble();
result.geo_extent.spatial = spatial_extent;
wtss::cxx::temporal_extent_t temporal_extent;
temporal_extent.resolution =
j_geo_extent["temporal"]["resolution"].GetInt64();
temporal_extent.time_interval.start =
j_geo_extent["temporal"]["start"].GetString();
temporal_extent.time_interval.end =
j_geo_extent["temporal"]["end"].GetString();
temporal_extent.unit = wtss::cxx::unit_t::from_string(
j_geo_extent["temporal"]["unit"].GetString());
result.geo_extent.temporal = temporal_extent;
return result;
}
wtss::cxx::geoarray_t wtss::cxx::client::describe_coverage(
const std::string& cv_name) const
{
std::string query_string("/describe_coverage?name=" + cv_name);
rapidjson::Document doc;
doc.Parse<0>(wtss::cxx::request(
wtss::cxx::client::server_uri + query_string).c_str());
if (doc.HasParseError())
{
boost::format err_msg("Error parsing requested document '%1%%2%': %3%.");
throw parse_error() << error_description(
(err_msg % server_uri % query_string %doc.GetParseError()).str());
}
if (!doc.IsObject())
throw parse_error() << error_description(
"Invalid JSON document: expecting a object!");
wtss::cxx::geoarray_t result;
if (!doc.HasMember("name"))
throw parse_error() << error_description(
"Invalid JSON document: expecting a member named \"coverage\"!");
result.name = doc["name"].GetString();
if (!doc.HasMember("description"))
throw parse_error() << error_description(
"Invalid JSON document: expecting a member named \"description\"!");
result.description = doc["description"].GetString();
if (!doc.HasMember("detail"))
throw parse_error() << error_description(
"Invalid JSON document: expecting a member named \"detail\"!");
result.detail = doc["detail"].GetString();
if (!doc.HasMember("dimensions"))
throw parse_error() << error_description(
"Invalid JSON document: expecting a member named \"dimensions\"!");
const rapidjson::Value& j_dimensions = doc["dimensions"];
if (!doc.HasMember("attributes"))
throw parse_error() << error_description(
"Invalid JSON document: expecting a member named \"attributes\"!");
const rapidjson::Value& j_attributes = doc["attributes"];
if (!j_dimensions.IsObject())
throw parse_error() << error_description(
"Invalid JSON document: member named \"dimensions\" must be an object!");
if (!j_attributes.IsArray())
throw parse_error() << error_description(
"Invalid JSON document: member named \"attributes\" must be an array!");
for (rapidjson::Value::ConstMemberIterator itr = j_dimensions.MemberBegin();
itr != j_dimensions.MemberEnd(); ++itr)
{
wtss::cxx::dimension_t dimension;
dimension.name = itr->value["name"].GetString();
dimension.max_idx = itr->value["max_idx"].GetInt64();
dimension.min_idx = itr->value["min_idx"].GetInt64();
result.dimensions.push_back(dimension);
}
for (rapidjson::Value::ConstValueIterator itr = j_attributes.Begin();
itr != j_attributes.End(); ++itr)
{
wtss::cxx::attribute_t attribute;
attribute.name = (*itr)["name"].GetString();
attribute.description = (*itr)["description"].GetString();
attribute.scale_factor = (*itr)["scale_factor"].GetDouble();
attribute.missing_value = (*itr)["missing_value"].GetDouble();
attribute.datatype =
wtss::cxx::datatype_t::from_string((*itr)["datatype"].GetString());
attribute.valid_range.max_val = (*itr)["valid_range"]["max"].GetDouble();
attribute.valid_range.min_val = (*itr)["valid_range"]["min"].GetDouble();
result.attributes.push_back(attribute);
}
wtss::cxx::spatial_extent_t spatial_extent;
spatial_extent.extent.xmin =
doc["spatial_extent"]["xmin"].GetDouble();
spatial_extent.extent.xmax =
doc["spatial_extent"]["xmax"].GetDouble();
spatial_extent.extent.ymin =
doc["spatial_extent"]["ymin"].GetDouble();
spatial_extent.extent.ymax =
doc["spatial_extent"]["ymax"].GetDouble();
spatial_extent.resolution.x =
doc["spatial_resolution"]["x"].GetDouble();
spatial_extent.resolution.y =
doc["spatial_resolution"]["y"].GetDouble();
result.spatial_extent = spatial_extent;
if (!doc.HasMember("crs"))
throw parse_error() << error_description(
"Invalid JSON document: expecting a member named \"crs\"!");
result.crs.proj4 = doc["crs"]["proj4"].GetString();
result.crs.wkt = doc["crs"]["wkt"].GetString();
if (!doc.HasMember("timeline"))
throw parse_error() << error_description(
"Invalid JSON document: expecting a member named \"timeline\"!");
const rapidjson::Value& j_timeline = doc["timeline"];
if (!j_timeline.IsArray())
throw parse_error() << error_description(
"Invalid JSON document: member named \"timeline\" must be an array!");
for (rapidjson::Value::ConstValueIterator itr = j_timeline.Begin();
itr != j_timeline.End(); ++itr)
{
const rapidjson::Value& j_date = (*itr);
std::vector<std::string> date_split;
std::string timeline = j_date.GetString();
std::string delimiter = "-";
size_t pos = 0;
std::string token;
while ((pos = timeline.find(delimiter)) != std::string::npos)
{
token = timeline.substr(0, pos);
date_split.push_back(token);
timeline.erase(0, pos + delimiter.length());
}
date_split.push_back(timeline);
date d;
if(date_split.size() > 0) d.year = std::stoi(date_split[0]);
if(date_split.size() > 1) d.month = std::stoi(date_split[1]);
if(date_split.size() > 2) d.day = std::stoi(date_split[2]);
result.timeline.push_back(d);
}
return result;
}
long string_input_popup::query_long( const bool loop, const bool draw_only )
{
return std::atol( query_string( loop, draw_only ).c_str() );
}
void string_input_popup::query( const bool loop, const bool draw_only )
{
query_string( loop, draw_only );
}
int main() {
clock_t start_time = std::clock();
snap::web::print_header();
// get user input
int content_length = atoi(getenv("CONTENT_LENGTH"));
char *input = new char[content_length+1];
fgets(input, content_length+1, stdin);
std::string query_string(input);
delete[] input;
// process user input
std::map<std::string, std::string> arguments = snap::web::parse_query_string(query_string);
std::string search_string01 = snap::web::decode_uri(boost::algorithm::trim_copy(arguments["search-string-1"]));
std::string search_string02 = snap::web::decode_uri(boost::algorithm::trim_copy(arguments["search-string-2"]));
int distance = stoi(arguments["distance"]);
boost::gregorian::date current_date, from_date, to_date;
try {
current_date = snap::date::string_to_date(arguments["from-date"]);
from_date = snap::date::string_to_date(arguments["from-date"]);
to_date = snap::date::string_to_date(arguments["to-date"]);
} catch (snap::date::InvalidDateException &e) {
const char *error_msg = e.what();
std::cout << "<span class=\"error\">" << error_msg << "</span>" << std::endl;
delete[] error_msg;
exit(-1);
}
int num_excerpts = stoi(arguments["num-excerpts"]);
int excerpt_size = stoi(arguments["excerpt-size"]);
std::vector<std::string> file_list = snap::io::generate_file_names(from_date, to_date, prefix, suffix);
std::vector<snap::Expression> expressions;
try {
expressions.emplace_back(search_string01);
expressions.emplace_back(search_string02);
} catch(snap::ExpressionSyntaxError &e) {
std::cout << "<span class=\"error\">" << e.what() << "</span>" << std::endl;
exit(-1);
}
std::vector<std::string> patterns;
for (auto it = expressions.begin(); it != expressions.end(); ++it) {
patterns.insert(patterns.end(), (it -> patterns).begin(), (it -> patterns).end());
}
// display user input
std::cout << "<p>" << std::endl;
std::cout << "Search string 1: " << search_string01 << "<br/>" << std::endl;
std::cout << "Search string 2: " << search_string02 << "<br/>" << std::endl;
std::cout << "From (inclusive): " << arguments["from-date"] << "<br/>" << std::endl;
std::cout << "To (inclusive): " << arguments["to-date"] << "<br/>" << std::endl;
std::cout << "Distance: " << arguments["distance"] << "<br/>" << std::endl;
std::cout << "Number of Excerpts: " << arguments["num-excerpts"] << "<br/>" << std::endl;
std::cout << "Excerpt Size: " << arguments["excerpt-size"] << "<br/>" << std::endl;
std::cout << "</p>" << std::endl;
// begin to iteratively process files
int matching_programs_sum = 0;
int total_matches01_sum = 0;
int total_matches02_sum = 0;
int selected_programs_sum = 0;
int total_programs_sum = 0;
std::vector<std::string> corrupt_files;
std::vector<snap::Excerpt> excerpts;
std::cout << "<pre>" << std::endl;
std::cout << "mt_prg = matching_programs_cnt" << '\n';
std::cout << "tot_mt1 = total_matches_cnt1" << '\n';
std::cout << "tot_mt2 = total_matches_cnt1" << '\n';
std::cout << "sel_prg = selected_programs_cnt" << '\n';
std::cout << "tot_prg = total_programs_cnt" << '\n';
// std::cout << "dt\tmatching_programs_cnt\ttotal_matches_cnt1\ttotal_matches_cnt2\tselected_programs_cnt\ttotal_programs_cnt" << std::endl;
std::cout << "dt \tmt_prg\ttot_mt1\ttot_mt2\tsel_prg\ttot_prg" << std::endl;
for (auto it = file_list.begin();
it != file_list.end();
++it) {
if (snap::io::file_exists(*it)) {
std::vector<snap::Program> programs;
try {
programs = snap::io::parse_programs(*it);
} catch (snap::io::CorruptFileException &e) {
programs.clear();
current_date += boost::gregorian::date_duration(1);
corrupt_files.push_back(*it);
continue;
}
std::cout << snap::date::date_to_string(current_date);
int matching_programs = 0;
int total_matches01 = 0; int total_matches02 = 0;
for (auto p = programs.begin(); p != programs.end(); ++p) {
std::map<std::string, std::vector<int>> match_positions;
match_positions = snap::near(expressions[0], expressions[1],
distance, p -> lower_text);
if (match_positions[expressions[0].raw_expression].size() > 0 || match_positions[expressions[1].raw_expression].size() > 0) {
++matching_programs;
total_matches01 += match_positions[expressions[0].raw_expression].size();
total_matches02 += match_positions[expressions[1].raw_expression].size();
for (auto it = match_positions[expressions[0].raw_expression].begin(); it != match_positions[expressions[0].raw_expression].end(); ++it) {
excerpts.emplace_back(*p, *it-distance-excerpt_size, *it+distance+excerpt_size);
for (auto pattern = patterns.begin(); pattern != patterns.end(); ++pattern) {
excerpts.back().highlight_word(*pattern);
}
}
}
match_positions.clear();
}
matching_programs_sum += matching_programs;
total_matches01_sum += total_matches01;
total_matches02_sum += total_matches02;
selected_programs_sum += programs.size();
total_programs_sum += programs.size();
std::cout << '\t' << matching_programs;
std::cout << '\t' << total_matches01;
std::cout << '\t' << total_matches02;
std::cout << '\t' << programs.size();
std::cout << '\t' << programs.size() << std::endl;
programs.clear();
}
current_date += boost::gregorian::date_duration(1);
}
std::cout << "Grand Total:";
std::cout << '\t' << matching_programs_sum;
std::cout << '\t' << total_matches01_sum;
std::cout << '\t' << total_matches02_sum;
std::cout << '\t' << selected_programs_sum;
std::cout << '\t' << total_programs_sum << std::endl;
std::cout << "</pre>" << std::endl;
snap::web::print_corrupt_files(corrupt_files);
// print excerpts
snap::web::print_excerpts(excerpts, num_excerpts, true);
double duration = (std::clock() - start_time) / (double) CLOCKS_PER_SEC;
std::cout << "<br/><span>Time taken (seconds): " << duration << "</span><br/>" << std::endl;
snap::web::close_html();
return 0;
}
int main() {
clock_t start_time = std::clock();
snap::web::print_header();
// get user input
int content_length = atoi(getenv("CONTENT_LENGTH"));
char *input = new char[content_length+1];
fgets(input, content_length+1, stdin);
std::string query_string(input);
delete[] input;
// process user input
std::map<std::string, std::string> arguments = snap::web::parse_query_string(query_string);
int num_excerpts = stoi(arguments["num-excerpts"]);
int excerpt_size = stoi(arguments["excerpt-size"]);
// dates
boost::gregorian::date current_date, from_date, to_date;
try {
current_date = snap::date::string_to_date(arguments["from-date"]);
from_date = snap::date::string_to_date(arguments["from-date"]);
to_date = snap::date::string_to_date(arguments["to-date"]);
} catch (snap::date::InvalidDateException &e) {
std::cout << "<span class=\"error\">" << e.what() << "</span>" << std::endl;
exit(-1);
}
std::vector<std::string> file_list = snap::io::generate_file_names(from_date, to_date, prefix, suffix);
// process search strings
std::vector<std::string> search_strings;
arguments["search-strings"] = snap::web::decode_uri(arguments["search-strings"]);
boost::split(search_strings, arguments["search-strings"], boost::is_any_of("\n"));
// remove empty strings
auto search_string_iterator = search_strings.begin();
while (search_string_iterator != search_strings.end()) {
if (std::all_of(search_string_iterator -> begin() , search_string_iterator -> end(), ::isspace)) {
search_string_iterator = search_strings.erase(search_string_iterator);
} else {
boost::algorithm::trim(*search_string_iterator);
++search_string_iterator;
}
}
if (search_strings.size() == 0) {
std::cout << "<span class=\"error\">" << "Error: There are no search strings." << "</span>" << std::endl;
exit(-1);
}
std::sort(search_strings.begin(), search_strings.end());
std::vector<snap::Expression> expressions;
std::set<std::string> pattern_set;
for (auto it = search_strings.begin(); it != search_strings.end(); ++it) {
try {
expressions.emplace_back(*it);
} catch(snap::ExpressionSyntaxError &e) {
const char *error_msg = e.what();
std::cout << "<span class=\"error\">" << error_msg << "</span>" << std::endl;
delete[] error_msg;
exit(-1);
}
pattern_set.insert(expressions.back().patterns.begin(), expressions.back().patterns.end());
}
std::vector<std::string> patterns;
patterns.insert(patterns.end(), pattern_set.begin(), pattern_set.end());
// print output for user to verify
std::cout << "<p>" << std::endl;
std::cout << "Search strings:" << "<br/>" << std::endl;
for (auto it = search_strings.begin(); it != search_strings.end(); ++it) {
std::cout << *it << "<br/>" << std::endl;
}
std::cout << "From (inclusive): <span id=\"from-date\">" << arguments["from-date"] << "</span><br/>" << std::endl;
std::cout << "To (inclusive): <span id=\"to-date\">" << arguments["to-date"] << "</span><br/>" << std::endl;
std::cout << "Number of Excerpts: " << arguments["num-excerpts"] << "<br/>" << std::endl;
std::cout << "Excerpt Size: " << arguments["excerpt-size"] << "<br/>" << std::endl;
std::cout << "</p>" << std::endl;
// variables to store results of loop
std::vector<std::vector<std::string>> search_results;
std::vector<std::vector<std::string>> search_results_programs;
std::vector<std::vector<std::string>> search_results_total_matches;
int total_programs_cnt = 0;
int selected_programs_cnt = 0;
std::vector<std::string> corrupt_files;
std::vector<std::string> missing_files;
std::vector<snap::Excerpt> excerpts;
// print table header
std::cout << "<table><thead><tr><th>dt</th>";
for (auto it = search_strings.begin(); it != search_strings.end(); ++it) {
std::cout << "<th>" << (*it) + " Contexts" << "</th>";
}
std::cout << "<th>selected_programs_cnt</th></tr></thead><tbody>" << std::endl;
snap::StringHasher hasher("", M, A);
std::unordered_map<std::string, std::unordered_map<int, int>> total_left_word_hashes;
std::unordered_map<std::string, std::unordered_map<int, int>> total_right_word_hashes;
std::map<std::string, std::tuple<int, int, int>> match_counts;
for (auto it = file_list.begin();
it != file_list.end();
++it) {
boost::gregorian::date current_date = snap::date::string_to_date((*it).substr(prefix.length(), 10));
if (snap::io::file_exists(*it)) {
std::vector<snap::Program> programs;
try {
programs = snap::io::parse_programs(*it);
} catch (snap::io::CorruptFileException &e) {
programs.clear();
corrupt_files.push_back(*it);
continue;
}
search_results.push_back(std::vector<std::string>{snap::date::date_to_string(current_date)});
search_results_programs.push_back(std::vector<std::string>{snap::date::date_to_string(current_date)});
search_results_total_matches.push_back(std::vector<std::string>{snap::date::date_to_string(current_date)});
std::unordered_map<std::string, std::unordered_map<int, int>> daily_left_word_hashes;
std::unordered_map<std::string, std::unordered_map<int, int>> daily_right_word_hashes;
std::cout << "<tr><td>" << snap::date::date_to_string(current_date) << "</td>";
total_programs_cnt += programs.size();
int daily_selected_programs_cnt = 0;
std::map<std::string, std::tuple<int, int, int>> daily_match_counts;
for (auto p = programs.begin(); p != programs.end(); ++p) {
++selected_programs_cnt;
++daily_selected_programs_cnt;
hasher.load_text(p -> lower_text);
std::map<std::string, std::vector<int>> raw_match_positions = snap::find(patterns, p -> lower_text);
std::map<std::string, std::vector<int>> match_positions = snap::evaluate_expressions(expressions, raw_match_positions);
for (auto ss = search_strings.begin(); ss != search_strings.end(); ++ss) {
if (match_positions[*ss].size() > 0) {
bool total_context_added = false;
bool context_added = false;
++std::get<1>(daily_match_counts[*ss]);
++std::get<1>(match_counts[*ss]);
std::get<2>(daily_match_counts[*ss]) += match_positions[*ss].size();
std::get<2>(match_counts[*ss]) += match_positions[*ss].size();
for (auto it = match_positions[*ss].begin(); it != match_positions[*ss].end(); ++it) {
int left_word_hash = hasher.hash(*it - LEFT_HASH_WIDTH, *it);
int right_word_hash = hasher.hash(*it, *it + RIGHT_HASH_WIDTH);
int daily_left_hash_cnt = daily_left_word_hashes[*ss][left_word_hash]++;
int daily_right_hash_cnt = daily_right_word_hashes[*ss][right_word_hash]++;
int total_left_hash_cnt = total_left_word_hashes[*ss][left_word_hash]++;
int total_right_hash_cnt = total_right_word_hashes[*ss][right_word_hash]++;
if (daily_left_hash_cnt == 0 && daily_right_hash_cnt == 0) {
if (!context_added) {
++std::get<0>(daily_match_counts[*ss]);
context_added = true;
}
if (total_left_hash_cnt == 0 && total_right_hash_cnt == 0) {
if (!total_context_added) {
++std::get<0>(match_counts[*ss]);
total_context_added = true;
}
excerpts.emplace_back(*p, *it - excerpt_size, *it + excerpt_size);
std::vector<std::string> search_string_patterns = expressions[ss - search_strings.begin()].patterns;
for (auto pattern = search_string_patterns.begin(); pattern != search_string_patterns.end(); ++pattern) {
excerpts.back().highlight_word(*pattern);
}
}
}
}
}
}
}
for (auto ss = search_strings.begin(); ss != search_strings.end(); ++ss) {
search_results.back().push_back(std::to_string(std::get<0>(daily_match_counts[*ss])));
search_results_programs.back().push_back(std::to_string(std::get<1>(daily_match_counts[*ss])));
search_results_total_matches.back().push_back(std::to_string(std::get<2>(daily_match_counts[*ss])));
std::cout << "<td>" << std::get<0>(daily_match_counts[*ss]) << "</td>";
}
search_results.back().push_back(std::to_string(daily_selected_programs_cnt));
search_results_programs.back().push_back(std::to_string(daily_selected_programs_cnt));
search_results_total_matches.back().push_back(std::to_string(daily_selected_programs_cnt));
std::cout << "<td>" << daily_selected_programs_cnt << "</td>";
std::cout << "</tr>" << std::endl;
programs.clear();
} else {
missing_files.push_back(*it);
}
}
// print out total line
std::cout << "<tr>" << std::endl;
search_results.emplace_back();
search_results_programs.emplace_back();
search_results_total_matches.emplace_back();
std::cout << "<td><strong>Grand Total:</strong></td>" << std::endl;
search_results.back().push_back("Grand Total:");
search_results_programs.back().push_back("Grand Total:");
search_results_total_matches.back().push_back("Grand Total:");
for (std::string ss : search_strings) {
std::cout << "<td>" << std::get<0>(match_counts[ss]) << "</td>" << std::endl;
search_results.back().push_back(std::to_string(std::get<0>(match_counts[ss])));
search_results_programs.back().push_back(std::to_string(std::get<1>(match_counts[ss])));
search_results_total_matches.back().push_back(std::to_string(std::get<2>(match_counts[ss])));
}
std::cout << "<td>" << total_programs_cnt << "</td>" << std::endl;
search_results.back().push_back(std::to_string(total_programs_cnt));
std::cout << "</tr>" << std::endl;
std::cout << "</tbody></table>" << std::endl;
std::cout << "<div>";
std::cout << "<br/>" << std::endl;
snap::web::print_missing_files(missing_files);
std::cout << "<br/>" << std::endl;
snap::web::print_corrupt_files(corrupt_files);
std::cout << "</div>" << std::endl;
snap::web::print_excerpts(excerpts, num_excerpts, true);
// output file
srand(time(NULL));
std::string random_id = std::to_string(rand());
output_matrix_file(search_results, search_strings, random_id, "contexts");
output_matrix_file(search_results_programs, search_strings, random_id, "programs");
output_matrix_file(search_results_total_matches, search_strings, random_id, "total_matches");
// all data in long form
std::map<std::string, std::tuple<std::string, std::string, std::string>> dict;
if (snap::io::file_exists("dictionary.csv")) {
std::ifstream dict_file("dictionary.csv");
dict = snap::io::read_dictionary(dict_file);
}
std::string output_file_name = search_results.front().front() + "_all_" + random_id + ".csv";
std::string output_file_path = output_path + output_file_name;
std::ofstream output_file(output_file_path);
output_file << "Date,Term,Contexts,Programs,Total Matches";
for (int i = 0; i < search_results.size() - 1; ++i) { // skip total line
for (int j = 0; j < search_strings.size(); ++j) {
output_file << '\n';
output_file << search_results[i].front() << ','
<< (dict.count(search_strings[j]) ? std::get<1>(dict[search_strings[j]]) : search_strings[j]) << ','
<< search_results[i][j + 1] << ',' // j + 1 skips date column
<< search_results_programs[i][j + 1] << ','
<< search_results_total_matches[i][j + 1];
}
}
output_file.close();
std::cout << "<p>";
std::cout << snap::web::create_link(output_file_path, "Output Long File", "long-data");
std::cout << "</p>" << std::endl;
std::cout << "<p>";
std::cout << snap::web::create_link("../time-series.html?filename=tmp%2F" + output_file_name + "&title=Snapstream%20Time%20Series",
"Visualization", "visualization");
std::cout << "</p>" << std::endl;
double duration = (std::clock() - start_time) / (double) CLOCKS_PER_SEC;
std::cout << "<br/><span>Time taken (seconds): " << duration << "</span><br/>" << std::endl;
snap::web::close_html();
return 0;
}