bool loadRaw(std::string const& data, std::string const& filename, char delimiter)
{
if (!loadCSV(data, delimiter))
return false;
currentDoc().filename_ = filename;
currentDoc().readOnly_ = true;
return true;
}
bool load(std::string const& filename)
{
std::ifstream file(filename.c_str());
if (!file.is_open())
{
logError("Could not open document '", filename, "'");
flashMessage("Could not open document!");
return false;
}
std::string data = "";
std::string line = "";
while (std::getline(file, line))
data += line + "\n";
if (data.size() == 0)
{
logError("No data in file '", filename, "'");
return false;
}
// Determin if we are reading a zum file, of a csv type of file.
if (data.size() > 5 && data[0] == 'Z' && data[1] == 'U' && data[2] == 'M' && data[3] == '1' && data[4] == '\n')
{
if (!loadZum1(data))
{
logError("Could not parse document '", filename, "'");
return false;
}
}
else
{
if (!loadCSV(data, 0))
{
logError("Could not parse document '", filename, "'");
return false;
}
}
currentDoc().filename_ = filename;
currentDoc().readOnly_ = false;
return true;
}
// Helper function to read a CSV file into an Eigen::MatrixXd.
static void loadEigenMatrixXdCSV(std::string file_name, Eigen::MatrixXd* matrix) {
CHECK_NOTNULL(matrix);
// Load CSV to matrix of strings
StringMatrix string_matrix = loadCSV(file_name);
CHECK_GE(string_matrix.size(), 1) << "CSV " << file_name << "was empty.";
// Iterate over the rows of the CSV and use comma-separated fields to populate outputs.
const unsigned n_rows = string_matrix.size();
const unsigned n_cols = string_matrix[0].size();
matrix->resize(n_rows, n_cols);
for (size_t i = 0u; i < n_rows; ++i) {
for (size_t j = 0u; j < n_cols; ++j) {
(*matrix)(i,j) = atof(string_matrix[i][j].c_str());
}
}
LOG(INFO) << "Loaded " << file_name << " with " << n_rows << " rows and " <<
n_cols << " cols.";
}
int main(int argc,char **argv) {
if (argc < 3)
{
cout << "Usage: " << argv[0] << " [CSV FILE] [FASTQ/GZ]" << "\n";
return EXIT_FAILURE;
}
int count_clean = 0, count_filter = 0;
string sequence = "";
string csv_file = argv[1];
string fastq_file = argv[2];
string out_clean_file = fastq_file + ".cleaned.result";
string out_filter_file = fastq_file + ".filtered.result";
FastQ f;
unordered_set<string> to_remove = loadCSV(csv_file);
struct magic_set *magic = magic_open(MAGIC_MIME|MAGIC_CHECK);
magic_load(magic, NULL);
// ifstream in_fastq(fastq_file, ios_base::in | ios_base::binary);
ifstream in_fastq(fastq_file, ios_base::in);
boost::iostreams::filtering_streambuf<boost::iostreams::input> in_buf;
if (std::strcmp("application/gzip; charset=binary", magic_file(magic, argv[2])) == 0)
in_buf.push(boost::iostreams::gzip_decompressor());
in_buf.push(in_fastq);
istream in_data(&in_buf);
ofstream out_clean(out_clean_file);
ofstream out_filter(out_filter_file);
cout << "\nUsing " << fastq_file << " and " << csv_file << "\n\nProcessing..." << "\n";
std::string buffer_filter;
buffer_filter.reserve(THRESHOLD);
std::string buffer_clean;
buffer_clean.reserve(THRESHOLD);
while(!in_data.eof())
{
if (!getline(in_data, f.name,'\n')) break;
if (!getline(in_data, f.sequence,'\n')) break;
if (!getline(in_data, f.info,'\n')) break;
if (!getline(in_data, f.quality,'\n')) break;
f.remove = false;
for (auto remove : to_remove) {
if (f.sequence.find(remove) != std::string::npos)
{
// cout << "Found: " << remove << " in " << f.name << '\n';
f.remove = true;
}
}
if (f.remove == false) {
if (buffer_clean.length() + 1 >= THRESHOLD) {
out_clean << buffer_clean;
buffer_clean.resize(0);
}
buffer_clean.append("@" + f.name + "\n");
buffer_clean.append(f.sequence + "\n");
buffer_clean.append(f.info + "\n");
buffer_clean.append(f.quality + "\n");
count_clean++;
}
else
{
if (buffer_filter.length() + 1 >= THRESHOLD) {
out_filter << buffer_filter;
buffer_filter.resize(0);
}
buffer_filter.append("@" + f.name + "\n");
buffer_filter.append(f.sequence + "\n");
buffer_filter.append(f.info + "\n");
buffer_filter.append(f.quality + "\n");
count_filter++;
}
}
out_clean << buffer_clean;
out_filter << buffer_filter;
cout << "\nCheck the results in " << out_clean_file << " (" << count_clean << ")\n";
cout << "\nFiltered results in " << out_filter_file << " (" << count_filter << ")\n";
out_clean.close();
out_filter.close();
in_fastq.close();
return EXIT_SUCCESS;
}
vector<double> classifyObjects(vector<MatDict > features)
{
// Load the SVM
svm_model *model = nullptr;
Mat train_max;
Mat train_min;
if (DEBUG) {
*model = *svm_load_model(MODEL_PATH);
train_max = loadCSV(TRAIN_MAX_PATH);
train_min = loadCSV(TRAIN_MIN_PATH);
} else {
#if !DEBUG
CFURLRef model_url = CFBundleCopyResourceURL(CFBundleGetMainBundle(),
CFSTR("model_out"), CFSTR("txt"),
NULL);
CFURLRef max_url = CFBundleCopyResourceURL(CFBundleGetMainBundle(),
CFSTR("train_max"), CFSTR("csv"),
NULL);
CFURLRef min_url = CFBundleCopyResourceURL(CFBundleGetMainBundle(),
CFSTR("train_min"), CFSTR("csv"),
NULL);
char model_path[1024];
char max_path[1024];
char min_path[1024];
CFURLGetFileSystemRepresentation(model_url, true,
model_path, sizeof(model_path));
CFURLGetFileSystemRepresentation(max_url, true,
max_path, sizeof(max_path));
CFURLGetFileSystemRepresentation(min_url, true,
min_path, sizeof(min_path));
CFRelease(model_url);
CFRelease(max_url);
CFRelease(min_url);
*model = *svm_load_model(model_path);
train_max = loadCSV(max_path);
train_min = loadCSV(min_path);
#endif
}
// Combine the features
cv::Mat featuresMatrix = cv::Mat((int)features.size(), 22, CV_64F);
vector<MatDict >::const_iterator it = features.begin();
int row = 0;
for (; it != features.end(); it++)
{
MatDict patch = *it;
cv::Mat geom = patch.find("geom")->second;
cv::Mat phi = patch.find("phi")->second;
int index = 0;
for (int i = 0; i < phi.rows; i++)
{
double val = phi.at<double>(i, 0);
featuresMatrix.at<double>(row, index) = val;
index++;
}
for (int i = 0; i < geom.rows; i++)
{
cv::Mat rowMat = patch.find("row")->second;
cv::Mat colMat = patch.find("col")->second;
float geom_val = geom.at<float>(i, 0);
featuresMatrix.at<double>(row, index) = (double) geom_val;
index++;
}
row++;
}
Debug::print(featuresMatrix, "xtest.txt");
// minmax normalization of features
cv::Mat maxMatrix = repMat(train_max, featuresMatrix.rows);
cv::Mat minMatrix = repMat(train_min, featuresMatrix.rows);
cv::Mat testMatrix = cv::Mat(featuresMatrix.rows, featuresMatrix.cols, featuresMatrix.type());
cv::Mat numerator = cv::Mat(featuresMatrix.rows, featuresMatrix.cols, featuresMatrix.type());
cv::Mat denominator = cv::Mat(featuresMatrix.rows, featuresMatrix.cols, featuresMatrix.type());
cv::subtract(featuresMatrix, minMatrix, numerator);
cv::subtract(maxMatrix, minMatrix, denominator);
cv::divide(numerator, denominator, testMatrix);
Debug::print(numerator, "numerator.txt");
Debug::print(testMatrix, "xtest_final.txt");
//testMatrix = Debug::loadMatrix("xtest_final.txt", 120, 22, CV_64F);
// Classify objects and get probabilities
vector<double> prob_results;
for (int i = 0; i < testMatrix.rows; i++) {
svm_node *node = new svm_node[testMatrix.cols + 2];
for (int j = 0; j < testMatrix.cols; j++) {
double d = testMatrix.at<double>(i, j);
node[j].index = j+1;
node[j].value = d;
}
node[testMatrix.cols].index = -1;
node[testMatrix.cols].value = 0;
double *probabilities = new double[2];
svm_predict_probability(model, node, probabilities);
prob_results.push_back(probabilities[0]);
}
cout << "Finished classifying features" << endl;
Debug::printVector(prob_results, "dvtest.txt");
return prob_results;
}
int main(int argc, char** argv){
int i;
/* setting of Point ID */
while(1){
time_t t = time(NULL);
for(i=0; i<REGISTER_COUNT; i++){
mydata[i].time = t;
strcpy(mydata[i].id,POINT_SET);
}
strcat(mydata[0].id,"vpv");
strcat(mydata[1].id,"ipv");
strcat(mydata[2].id,"ppv");
strcat(mydata[3].id,"qpv");
strcat(mydata[4].id,"whpv");
strcat(mydata[5].id,"fpv");
strcat(mydata[6].id,"pfpv");
strcat(mydata[7].id,"irpv");
strcat(mydata[8].id,"ptpv");
strcat(mydata[9].id,"tpv");
strcat(mydata[10].id,"vpdc");
strcat(mydata[11].id,"ipdc");
strcat(mydata[12].id,"ppdc");
strcat(mydata[13].id,"efpv");
strcat(mydata[14].id,"efpi");
strcat(mydata[15].id,"efpt");
loadCSV();
/*IEEE object create */
ieee1888_transport* request = ieee1888_mk_transport();
request -> body = ieee1888_mk_body();
ieee1888_pointSet* ps = ieee1888_mk_pointSet_array(1);
ps -> id = ieee1888_mk_uri("http://www.gutp.jp/hour/pv/");
request -> body -> pointSet = ps;
request -> body -> n_pointSet = 1;
ieee1888_point* p = ieee1888_mk_point_array(REGISTER_COUNT);
ps -> point = p;
ps -> n_point = REGISTER_COUNT;
/*putting into sendint date for object*/
for(i=0; i<REGISTER_COUNT; i++){
p[i].id = ieee1888_mk_uri(mydata[i].id);
p[i].value = ieee1888_mk_value();
p[i].n_value = 1;
p[i].value -> time = ieee1888_mk_time(mydata[i].time);
p[i].value -> content = ieee1888_mk_string(mydata[i].value);
}
int err = 0;
/*display querry and object*/
ieee1888_dump_objects((ieee1888_object*)request);
/*connecting with server*/
ieee1888_transport* response = ieee1888_client_data(request,"http://52.27.198.165/axis2/services/FIAPStorage",NULL,&err);
/*display response message and objects*/
ieee1888_dump_objects((ieee1888_object*)response);
/*success*/
if(response != NULL){
if(response -> header -> OK != NULL){
printf("Data Upload Success\n");
}else{
}
}
/*delete object in memory*/
if(request != NULL){
ieee1888_destroy_objects((ieee1888_object*)request);
free(request);
}
if(response != NULL){
ieee1888_destroy_objects((ieee1888_object*)response);
free(response);
}
sleep(3600);
}
return 0;
}
