TEST_F(LoaderTests, wrong_inputs_should_fail) { CSVInput input("somefile"); Loader::params p; p.setInput(input); ASSERT_THROW( { Loader::load(p); }, Loader::Error);
void StorageManager::loadTableFile(std::string name, std::string fileName) { CSVInput input(makePath(fileName)); CSVHeader header(makePath(fileName)); Loader::params p; p.setInput(input); p.setHeader(header); addStorageTable(name, p); }
std::shared_ptr<storage::AbstractTable> Loader::shortcuts::loadRaw(const std::string& file) { RawTableLoader input(file); CSVHeader header(file); Loader::params p; p.setInput(input); p.setHeader(header); p.setReturnsMutableVerticalTable(false); return Loader::load(p); }
Loader::params *Loader::params::clone() const { Loader::params *p = new Loader::params(); if (Input != nullptr) p->setInput(*Input); if (Header != nullptr) p->setHeader(*Header); p->setBasePath(BasePath); p->setFactory(Factory); p->setInsertOnly(InsertOnly); p->setReturnsMutableVerticalTable(ReturnsMutableVerticalTable); p->setModifiableMutableVerticalTable(ModifiableMutableVerticalTable); p->setReferenceTable(ReferenceTable); p->setCompressed(Compressed); return p; }
TEST_F(StringLoaderTests, load_test_typesafe) { hyrise::storage::atable_ptr_t t = Loader::load( Loader::params() .setHeader(StringHeader("employee_id|employee_company_id|employee_name\n" "INTEGER|INTEGER|STRING\n" "0_C | 0_C | 0_C")) .setInput(CSVInput("test/tables/employees.data")) ); EmptyInput input; StringHeader header("employee_id|employee_company_id|employee_name\n" "INTEGER|INTEGER|INTEGER\n" "0_C | 0_C | 0_C"); Loader::params p; p.setInput(input).setHeader(header).setReturnsMutableVerticalTable(true).setReferenceTable(t); auto res = Loader::load(p); ASSERT_EQ(t->typeOfColumn(2), res->typeOfColumn(2)); }
std::shared_ptr<storage::Store> Loader::shortcuts::loadMainDelta(const std::string& mainfilepath, const std::string& deltafilepath, Loader::params p) { std::vector<std::string> filenames; filenames.push_back(mainfilepath); filenames.push_back(deltafilepath); std::vector<std::shared_ptr<storage::AbstractTable>> tables; for (int i = 0; i < 2; ++i) { CSVInput input(filenames[i]); CSVHeader header(filenames[i]); p.setInput(input); p.setHeader(header); p.setReturnsMutableVerticalTable(true); std::shared_ptr<storage::AbstractTable> table = load(p); tables.push_back(table); } auto s = std::make_shared<storage::Store>(tables[0]); s->setDelta(tables[1]); return s; };
std::shared_ptr<AbstractTable> CSVInput::load(std::shared_ptr<AbstractTable> intable, const compound_metadata_list *meta, const Loader::params &args) { cb_data data(intable->columnCount(), _parameters.getUnsafe()); data.table = intable; csv::params params(_parameters.getCSVParams()); if (detectHeader(args.getBasePath() + _filename)) params.setLineStart(5); // Resize the table based on the file size data.table->resize(countLines(args.getBasePath() + _filename) - params.getLineStart() + 1); try { csv::genericParseFile(args.getBasePath() + _filename, (field_cb_t) cb_per_field, (line_cb_t) cb_per_line, &data, params); } catch (const csv::ParserError &e) { throw Loader::Error(e.what()); } data.table->resize(data.row); return intable; }
Loader::params::params(const Loader::params &other) : Factory(other.getFactory()), BasePath(other.getBasePath()), InsertOnly(other.getInsertOnly()), ModifiableMutableVerticalTable(other.getModifiableMutableVerticalTable()), ReturnsMutableVerticalTable(other.getReturnsMutableVerticalTable()), Compressed(other.getCompressed()) { if (other.Input != nullptr) Input = other.Input->clone(); if (other.Header != nullptr) Header = other.Header->clone(); if (other.ReferenceTable != nullptr) ReferenceTable = other.ReferenceTable; }
void StorageManager::loadTable(std::string name, const Loader::params ¶meters) { Loader::params *p = parameters.clone(); p->setBasePath(Settings::getInstance()->getDBPath() + "/"); addStorageTable(name, *p); delete p; }
std::shared_ptr<storage::AbstractTable> RawTableLoader::load(std::shared_ptr<storage::AbstractTable> in, const storage::compound_metadata_list *ml, const Loader::params &args) { csv::params params; if (detectHeader(args.getBasePath() + _filename)) params.setLineStart(5); // Create the result table storage::metadata_vec_t v(in->columnCount()); for(size_t i=0; i < in->columnCount(); ++i) { v[i] = in->metadataAt(i); } auto result = std::make_shared<storage::RawTable>(v); // CSV Parsing std::ifstream file(args.getBasePath() + _filename, std::ios::binary); if (!file || file.bad()) { throw csv::ParserError("CSV file '" + _filename + "' does not exist"); } struct csv_parser parser; if (!csv_init(&parser, 0)) { csv_set_opts(&parser, CSV_APPEND_NULL); csv_set_delim(&parser, params.getDelimiter()); // If there is a header in the file, we will ignore it std::string line; int line_start = params.getLineStart(); if (line_start != 1) { while (line_start > 1) { std::getline(file, line); --line_start; } } // Prepare cb data handler struct raw_table_cb_data data(v); data.table = result; const size_t block_size = 16 * 1024; char rdbuf [block_size]; while (file.read(rdbuf, block_size).good()) { auto extracted = file.gcount(); if (extracted == 0) break; if (csv_parse(&parser, rdbuf, extracted, (field_cb_t) raw_table_cb_per_field, (line_cb_t) raw_table_cb_per_line, (void*) &data) != (size_t) extracted) { throw csv::ParserError(csv_strerror(csv_error(&parser))); } } // Parse the rest if (csv_parse(&parser, rdbuf, file.gcount(), (field_cb_t) raw_table_cb_per_field, (line_cb_t) raw_table_cb_per_line, (void*) &data) != (size_t) file.gcount()) { throw csv::ParserError(csv_strerror(csv_error(&parser))); } csv_fini(&parser, (field_cb_t) raw_table_cb_per_field, (line_cb_t) raw_table_cb_per_line, (void*) &data); } csv_free(&parser); return result; }