std::pair<bool,typename T::value_type::first_type> process_csv_file(T & boxes, std::string const& filename, std::string const& manual_headers, char separator, char quote)
{
using box_type = typename T::value_type::first_type;
csv_utils::csv_file_parser p;
p.manual_headers_ = manual_headers;
p.separator_ = separator;
p.quote_ = quote;
#if defined(MAPNIK_MEMORY_MAPPED_FILE)
using file_source_type = boost::interprocess::ibufferstream;
file_source_type csv_file;
mapnik::mapped_region_ptr mapped_region;
boost::optional<mapnik::mapped_region_ptr> memory =
mapnik::mapped_memory_cache::instance().find(filename, true);
if (memory)
{
mapped_region = *memory;
csv_file.buffer(static_cast<char*>(mapped_region->get_address()),mapped_region->get_size());
}
else
{
std::clog << "Error : cannot mmap " << filename << std::endl;
return std::make_pair(false, box_type(p.extent_));
}
#else
#if defined(_WINDOWS)
fs::ifstream csv_file(mapnik::utf8_to_utf16(filename),std::ios_base::in | std::ios_base::binary);
#else
fs::ifstream csv_file(filename.c_str(),std::ios_base::in | std::ios_base::binary);
#endif
if (!csv_file.is_open())
{
std::clog << "Error : cannot open " << filename << std::endl;
return std::make_pair(false, box_type(p.extent_));
}
#endif
try
{
p.parse_csv_and_boxes(csv_file, boxes);
return std::make_pair(true, box_type(p.extent_));
}
catch (std::exception const& ex)
{
std::clog << ex.what() << std::endl;
return std::make_pair(false, box_type(p.extent_));
}
}
int write_black_report(std::map<std::string, std::vector<Real>>
blackReport, std::string name) {
std::ofstream csv_file(name);
std::vector<Real> v_prices = blackReport["Price"];
csv_file << "Price,Delta,Gamma,Theta,Vega,Rho\n";
for(int idx=0; idx<v_prices.size(); idx++) {
csv_file << v_prices[idx] << "," << blackReport["Delta"][idx]
<< "," << blackReport["Gamma"][idx]
<<","<< blackReport["Theta"][idx]
<< "," <<blackReport["Vega"][idx]
<< "," << blackReport["Rho"][idx]<<"\n";
}
csv_file.close();
return 1;
}
bool LoadCSV(const std::string& i_file_path, THandleLine i_line_handler)
{
std::string line;
std::ifstream csv_file(i_file_path);
if (csv_file.is_open())
{
while (std::getline(csv_file, line) )
{
TValues values;
_StringSplit(line, g_delimiter, values);
if (! i_line_handler(values))
return false;
}
csv_file.close();
return true;
}
else
{
return false;
}
}
void mmStockDialog::OnHistoryImportButton(wxCommandEvent& /*event*/)
{
if (m_stock->SYMBOL.IsEmpty())
return;
bool canceledbyuser = false;
const wxString fileName = wxFileSelector(_("Choose CSV data file to import")
, wxEmptyString, wxEmptyString, wxEmptyString, "*.csv", wxFD_FILE_MUST_EXIST);
Model_Account::Data *account = Model_Account::instance().get(m_stock->HELDAT);
Model_Currency::Data *currency = Model_Account::currency(account);
if (!fileName.IsEmpty())
{
wxFileName csv_file(fileName);
if (fileName.IsEmpty() || !csv_file.FileExists())
return;
wxTextFile tFile(fileName);
if (!tFile.Open())
return;
wxProgressDialog* progressDlg = new wxProgressDialog(_("Stock History CSV Import")
, _("Quotes imported from CSV: "), tFile.GetLineCount()
, NULL, wxPD_AUTO_HIDE | wxPD_APP_MODAL | wxPD_SMOOTH | wxPD_CAN_ABORT);
long countNumTotal = 0;
long countImported = 0;
double price;
wxDateTime dt;
wxString dateStr, priceStr;
Model_StockHistory::Data *data;
Model_StockHistory::Cache stockData;
wxString line;
std::vector<wxString> rows;
for (line = tFile.GetFirstLine(); !tFile.Eof(); line = tFile.GetNextLine())
{
wxString progressMsg;
progressMsg << _("Quotes imported from CSV: ") << countImported;
if (!progressDlg->Update(countImported, progressMsg))
{
canceledbyuser = true;
break; // abort processing
}
if (!line.IsEmpty())
++countNumTotal;
else
continue;
dateStr.clear();
priceStr.clear();
const wxString& delimiter = Model_Infotable::instance().GetStringInfo("DELIMITER", mmex::DEFDELIMTER);
mmUnivCSVDialog::csv2tab_separated_values(line, delimiter);
wxStringTokenizer tkz(line, "\t", wxTOKEN_RET_EMPTY_ALL);
if ((int)tkz.CountTokens() < 2)
continue;
std::vector<wxString> tokens;
while (tkz.HasMoreTokens())
{
wxString token = tkz.GetNextToken();
tokens.push_back(token);
}
// date
dateStr = tokens[0];
mmParseDisplayStringToDate(dt, dateStr, mmOptions::instance().dateFormat_);
dateStr = dt.FormatISODate();
// price
priceStr = tokens[1];
priceStr.Replace(" ", wxEmptyString);
if (!Model_Currency::fromString(priceStr, price, currency) || price <= 0.0)
continue;
data = Model_StockHistory::instance().create();
data->SYMBOL = m_stock->SYMBOL;
data->DATE = dateStr;
data->VALUE = price;
data->UPDTYPE = 2;
stockData.push_back(data);
if (rows.size()<10)
{
dateStr << wxT (" ") << priceStr;
rows.push_back(dateStr);
}
countImported++;
}
progressDlg->Destroy();
wxString msg = wxString::Format(_("Total Lines : %ld"), countNumTotal);
msg << "\n";
msg << wxString::Format(_("Total Imported : %ld"), countImported);
msg << "\n";
msg << _("Date") << " " << _("Price");
msg << "\n";
for (std::vector<wxString>::const_iterator d = rows.begin(); d != rows.end(); ++d)
msg << *d << "\n";
wxString confirmMsg = msg + _("Please confirm saving...");
if (!canceledbyuser && wxMessageBox(confirmMsg
, _("Importing CSV"), wxOK | wxCANCEL | wxICON_INFORMATION) == wxCANCEL)
{
canceledbyuser = true;
}
// Since all database transactions are only in memory,
if (!canceledbyuser)
{
// we need to save them to the database.
for (auto &d : stockData)
Model_StockHistory::instance().save(d);
// show the data
showStockHistory();
}
else
{
//TODO: and discard the database changes.
}
}
}
std::pair<bool,box2d<double>> process_csv_file(T & boxes, std::string const& filename, std::string const& manual_headers, char separator, char quote)
{
mapnik::box2d<double> extent;
#if defined(MAPNIK_MEMORY_MAPPED_FILE)
using file_source_type = boost::interprocess::ibufferstream;
file_source_type csv_file;
mapnik::mapped_region_ptr mapped_region;
boost::optional<mapnik::mapped_region_ptr> memory =
mapnik::mapped_memory_cache::instance().find(filename, true);
if (memory)
{
mapped_region = *memory;
csv_file.buffer(static_cast<char*>(mapped_region->get_address()),mapped_region->get_size());
}
else
{
std::clog << "Error : cannot mmap " << filename << std::endl;
return std::make_pair(false, extent);
}
#else
#if defined(_WINDOWS)
std::ifstream csv_file(mapnik::utf8_to_utf16(filename),std::ios_base::in | std::ios_base::binary);
#else
std::ifstream csv_file(filename.c_str(),std::ios_base::in | std::ios_base::binary);
#endif
if (!csv_file.is_open())
{
std::clog << "Error : cannot open " << filename << std::endl;
return std::make_pair(false, extent);
}
#endif
auto file_length = ::detail::file_length(csv_file);
// set back to start
csv_file.seekg(0, std::ios::beg);
char newline;
bool has_newline;
char detected_quote;
std::tie(newline, has_newline, detected_quote) = ::detail::autodect_newline_and_quote(csv_file, file_length);
if (quote == 0) quote = detected_quote;
// set back to start
csv_file.seekg(0, std::ios::beg);
// get first line
std::string csv_line;
csv_utils::getline_csv(csv_file, csv_line, newline, quote);
if (separator == 0) separator = ::detail::detect_separator(csv_line);
csv_file.seekg(0, std::ios::beg);
int line_number = 0;
::detail::geometry_column_locator locator;
std::vector<std::string> headers;
std::clog << "Parsing CSV using SEPARATOR=" << separator << " QUOTE=" << quote << std::endl;
if (!manual_headers.empty())
{
std::size_t index = 0;
headers = csv_utils::parse_line(manual_headers, separator, quote);
for (auto const& header : headers)
{
::detail::locate_geometry_column(header, index++, locator);
headers.push_back(header);
}
}
else // parse first line as headers
{
while (csv_utils::getline_csv(csv_file,csv_line,newline, quote))
{
try
{
headers = csv_utils::parse_line(csv_line, separator, quote);
// skip blank lines
if (headers.size() > 0 && headers[0].empty()) ++line_number;
else
{
std::size_t index = 0;
for (auto & header : headers)
{
mapnik::util::trim(header);
if (header.empty())
{
// create a placeholder for the empty header
std::ostringstream s;
s << "_" << index;
header = s.str();
}
else
{
::detail::locate_geometry_column(header, index, locator);
}
++index;
}
++line_number;
break;
}
}
catch (std::exception const& ex)
{
std::string s("CSV index: error parsing headers: ");
s += ex.what();
std::clog << s << std::endl;
return std::make_pair(false, extent);
}
}
}
std::size_t num_headers = headers.size();
if (!::detail::valid(locator, num_headers))
{
std::clog << "CSV index: could not detect column(s) with the name(s) of wkt, geojson, x/y, or "
<< "latitude/longitude in:\n"
<< csv_line
<< "\n - this is required for reading geometry data"
<< std::endl;
return std::make_pair(false, extent);
}
auto pos = csv_file.tellg();
// handle rare case of a single line of data and user-provided headers
// where a lack of a newline will mean that csv_utils::getline_csv returns false
bool is_first_row = false;
if (!has_newline)
{
csv_file.setstate(std::ios::failbit);
pos = 0;
if (!csv_line.empty())
{
is_first_row = true;
}
}
while (is_first_row || csv_utils::getline_csv(csv_file, csv_line, newline, quote))
{
++line_number;
auto record_offset = pos;
auto record_size = csv_line.length();
pos = csv_file.tellg();
is_first_row = false;
// skip blank lines
if (record_size <= 10)
{
std::string trimmed = csv_line;
boost::trim_if(trimmed, boost::algorithm::is_any_of("\",'\r\n "));
if (trimmed.empty())
{
std::clog << "CSV index: empty row encountered at line: " << line_number << std::endl;
continue;
}
}
try
{
auto const* start_line = csv_line.data();
auto const* end_line = start_line + csv_line.size();
auto values = csv_utils::parse_line(start_line, end_line, separator, quote, num_headers);
unsigned num_fields = values.size();
if (num_fields != num_headers)
{
std::ostringstream s;
s << "CSV Plugin: # of columns(" << num_fields << ")";
if (num_fields > num_headers)
{
s << " > ";
}
else
{
s << " < ";
}
s << "# of headers(" << num_headers << ") parsed";
throw mapnik::datasource_exception(s.str());
}
auto geom = ::detail::extract_geometry(values, locator);
if (!geom.is<mapnik::geometry::geometry_empty>())
{
auto box = mapnik::geometry::envelope(geom);
if (!extent.valid()) extent = box;
else extent.expand_to_include(box);
boxes.emplace_back(std::move(box), make_pair(record_offset, record_size));
}
else
{
std::ostringstream s;
s << "CSV Index: expected geometry column: could not parse row "
<< line_number << " "
<< values[locator.index] << "'";
throw mapnik::datasource_exception(s.str());
}
}
catch (mapnik::datasource_exception const& ex )
{
std::clog << ex.what() << " at line: " << line_number << std::endl;
}
catch (std::exception const& ex)
{
std::ostringstream s;
s << "CSV Index: unexpected error parsing line: " << line_number
<< " - found " << headers.size() << " with values like: " << csv_line << "\n"
<< " and got error like: " << ex.what();
std::clog << s.str() << std::endl;
}
}
return std::make_pair(true, extent);;
}
