TEST_F(LoaderTests, wrong_inputs_should_fail) {
CSVInput input("somefile");
Loader::params p;
p.setInput(input);
ASSERT_THROW( {
Loader::load(p);
}, Loader::Error);
void StorageManager::loadTableFile(std::string name, std::string fileName) {
CSVInput input(makePath(fileName));
CSVHeader header(makePath(fileName));
Loader::params p;
p.setInput(input);
p.setHeader(header);
addStorageTable(name, p);
}
std::shared_ptr<storage::AbstractTable> Loader::shortcuts::loadRaw(const std::string& file) {
RawTableLoader input(file);
CSVHeader header(file);
Loader::params p;
p.setInput(input);
p.setHeader(header);
p.setReturnsMutableVerticalTable(false);
return Loader::load(p);
}
Loader::params *Loader::params::clone() const {
Loader::params *p = new Loader::params();
if (Input != nullptr) p->setInput(*Input);
if (Header != nullptr) p->setHeader(*Header);
p->setBasePath(BasePath);
p->setFactory(Factory);
p->setInsertOnly(InsertOnly);
p->setReturnsMutableVerticalTable(ReturnsMutableVerticalTable);
p->setModifiableMutableVerticalTable(ModifiableMutableVerticalTable);
p->setReferenceTable(ReferenceTable);
p->setCompressed(Compressed);
return p;
}
TEST_F(StringLoaderTests, load_test_typesafe) {
hyrise::storage::atable_ptr_t t = Loader::load(
Loader::params()
.setHeader(StringHeader("employee_id|employee_company_id|employee_name\n"
"INTEGER|INTEGER|STRING\n"
"0_C | 0_C | 0_C"))
.setInput(CSVInput("test/tables/employees.data"))
);
EmptyInput input;
StringHeader header("employee_id|employee_company_id|employee_name\n"
"INTEGER|INTEGER|INTEGER\n"
"0_C | 0_C | 0_C");
Loader::params p;
p.setInput(input).setHeader(header).setReturnsMutableVerticalTable(true).setReferenceTable(t);
auto res = Loader::load(p);
ASSERT_EQ(t->typeOfColumn(2), res->typeOfColumn(2));
}
std::shared_ptr<storage::Store> Loader::shortcuts::loadMainDelta(const std::string& mainfilepath,
const std::string& deltafilepath,
Loader::params p) {
std::vector<std::string> filenames;
filenames.push_back(mainfilepath);
filenames.push_back(deltafilepath);
std::vector<std::shared_ptr<storage::AbstractTable>> tables;
for (int i = 0; i < 2; ++i) {
CSVInput input(filenames[i]);
CSVHeader header(filenames[i]);
p.setInput(input);
p.setHeader(header);
p.setReturnsMutableVerticalTable(true);
std::shared_ptr<storage::AbstractTable> table = load(p);
tables.push_back(table);
}
auto s = std::make_shared<storage::Store>(tables[0]);
s->setDelta(tables[1]);
return s;
};
std::shared_ptr<AbstractTable> CSVInput::load(std::shared_ptr<AbstractTable> intable, const compound_metadata_list *meta, const Loader::params &args) {
cb_data data(intable->columnCount(), _parameters.getUnsafe());
data.table = intable;
csv::params params(_parameters.getCSVParams());
if (detectHeader(args.getBasePath() + _filename)) params.setLineStart(5);
// Resize the table based on the file size
data.table->resize(countLines(args.getBasePath() + _filename) - params.getLineStart() + 1);
try {
csv::genericParseFile(args.getBasePath() + _filename,
(field_cb_t) cb_per_field,
(line_cb_t) cb_per_line,
&data,
params);
} catch (const csv::ParserError &e) {
throw Loader::Error(e.what());
}
data.table->resize(data.row);
return intable;
}
Loader::params::params(const Loader::params &other) :
Factory(other.getFactory()),
BasePath(other.getBasePath()),
InsertOnly(other.getInsertOnly()),
ModifiableMutableVerticalTable(other.getModifiableMutableVerticalTable()),
ReturnsMutableVerticalTable(other.getReturnsMutableVerticalTable()),
Compressed(other.getCompressed()) {
if (other.Input != nullptr) Input = other.Input->clone();
if (other.Header != nullptr) Header = other.Header->clone();
if (other.ReferenceTable != nullptr) ReferenceTable = other.ReferenceTable;
}
void StorageManager::loadTable(std::string name, const Loader::params ¶meters) {
Loader::params *p = parameters.clone();
p->setBasePath(Settings::getInstance()->getDBPath() + "/");
addStorageTable(name, *p);
delete p;
}
std::shared_ptr<storage::AbstractTable> RawTableLoader::load(std::shared_ptr<storage::AbstractTable> in,
const storage::compound_metadata_list *ml,
const Loader::params &args) {
csv::params params;
if (detectHeader(args.getBasePath() + _filename)) params.setLineStart(5);
// Create the result table
storage::metadata_vec_t v(in->columnCount());
for(size_t i=0; i < in->columnCount(); ++i) {
v[i] = in->metadataAt(i);
}
auto result = std::make_shared<storage::RawTable>(v);
// CSV Parsing
std::ifstream file(args.getBasePath() + _filename, std::ios::binary);
if (!file || file.bad()) {
throw csv::ParserError("CSV file '" + _filename + "' does not exist");
}
struct csv_parser parser;
if (!csv_init(&parser, 0)) {
csv_set_opts(&parser, CSV_APPEND_NULL);
csv_set_delim(&parser, params.getDelimiter());
// If there is a header in the file, we will ignore it
std::string line;
int line_start = params.getLineStart();
if (line_start != 1) {
while (line_start > 1) {
std::getline(file, line);
--line_start;
}
}
// Prepare cb data handler
struct raw_table_cb_data data(v);
data.table = result;
const size_t block_size = 16 * 1024;
char rdbuf [block_size];
while (file.read(rdbuf, block_size).good()) {
auto extracted = file.gcount();
if (extracted == 0)
break;
if (csv_parse(&parser,
rdbuf,
extracted,
(field_cb_t) raw_table_cb_per_field,
(line_cb_t) raw_table_cb_per_line,
(void*) &data) != (size_t) extracted) {
throw csv::ParserError(csv_strerror(csv_error(&parser)));
}
}
// Parse the rest
if (csv_parse(&parser,
rdbuf,
file.gcount(),
(field_cb_t) raw_table_cb_per_field,
(line_cb_t) raw_table_cb_per_line,
(void*) &data) != (size_t) file.gcount()) {
throw csv::ParserError(csv_strerror(csv_error(&parser)));
}
csv_fini(&parser,
(field_cb_t) raw_table_cb_per_field,
(line_cb_t) raw_table_cb_per_line,
(void*) &data);
}
csv_free(&parser);
return result;
}
