Beispiel #1
0
TEST_F(LoaderTests, wrong_inputs_should_fail) {
  CSVInput input("somefile");
  Loader::params p;
  p.setInput(input);
  ASSERT_THROW( {
      Loader::load(p);
    }, Loader::Error);
Beispiel #2
0
void StorageManager::loadTableFile(std::string name, std::string fileName) {
  CSVInput input(makePath(fileName));
  CSVHeader header(makePath(fileName));
  Loader::params p;
  p.setInput(input);
  p.setHeader(header);
  addStorageTable(name, p);
}
Beispiel #3
0
std::shared_ptr<storage::AbstractTable> Loader::shortcuts::loadRaw(const std::string& file) {
  RawTableLoader input(file);
  CSVHeader header(file);

  Loader::params p;
  p.setInput(input);
  p.setHeader(header);
  p.setReturnsMutableVerticalTable(false);
  return Loader::load(p);
}
Beispiel #4
0
Loader::params *Loader::params::clone() const {
  Loader::params *p = new Loader::params();
  if (Input != nullptr) p->setInput(*Input);
  if (Header != nullptr) p->setHeader(*Header);
  p->setBasePath(BasePath);
  p->setFactory(Factory);
  p->setInsertOnly(InsertOnly);
  p->setReturnsMutableVerticalTable(ReturnsMutableVerticalTable);
  p->setModifiableMutableVerticalTable(ModifiableMutableVerticalTable);
  p->setReferenceTable(ReferenceTable);
  p->setCompressed(Compressed);
  return p;
}
Beispiel #5
0
TEST_F(StringLoaderTests, load_test_typesafe) {
  hyrise::storage::atable_ptr_t  t = Loader::load(
      Loader::params()
      .setHeader(StringHeader("employee_id|employee_company_id|employee_name\n"
                              "INTEGER|INTEGER|STRING\n"
                              "0_C | 0_C | 0_C"))
      .setInput(CSVInput("test/tables/employees.data"))
                                                  );


  EmptyInput input;
  StringHeader header("employee_id|employee_company_id|employee_name\n"
                      "INTEGER|INTEGER|INTEGER\n"
                      "0_C | 0_C | 0_C");

  Loader::params p;
  p.setInput(input).setHeader(header).setReturnsMutableVerticalTable(true).setReferenceTable(t);

  auto res = Loader::load(p);

  ASSERT_EQ(t->typeOfColumn(2), res->typeOfColumn(2));
}
Beispiel #6
0
std::shared_ptr<storage::Store> Loader::shortcuts::loadMainDelta(const std::string& mainfilepath,
                                                                 const std::string& deltafilepath,
                                                                 Loader::params p) {
  std::vector<std::string> filenames;
  filenames.push_back(mainfilepath);
  filenames.push_back(deltafilepath);
  std::vector<std::shared_ptr<storage::AbstractTable>> tables;

  for (int i = 0; i < 2; ++i) {
    CSVInput input(filenames[i]);
    CSVHeader header(filenames[i]);

    p.setInput(input);
    p.setHeader(header);
    p.setReturnsMutableVerticalTable(true);
    std::shared_ptr<storage::AbstractTable> table = load(p);
    tables.push_back(table);
  }
  auto s = std::make_shared<storage::Store>(tables[0]);
  s->setDelta(tables[1]);
  return s;
};
Beispiel #7
0
std::shared_ptr<AbstractTable> CSVInput::load(std::shared_ptr<AbstractTable> intable, const compound_metadata_list *meta, const Loader::params &args) {
  cb_data data(intable->columnCount(), _parameters.getUnsafe());
  data.table = intable;
  csv::params params(_parameters.getCSVParams());

  if (detectHeader(args.getBasePath() + _filename)) params.setLineStart(5);

  // Resize the table based on the file size
  data.table->resize(countLines(args.getBasePath() + _filename) - params.getLineStart() + 1);

  try {
    csv::genericParseFile(args.getBasePath() + _filename,
                          (field_cb_t) cb_per_field,
                          (line_cb_t) cb_per_line,
                          &data,
                          params);
  } catch (const csv::ParserError &e) {
    throw Loader::Error(e.what());
  }

  data.table->resize(data.row);
  return intable;
}
Beispiel #8
0
Loader::params::params(const Loader::params &other) :
  Factory(other.getFactory()),
  BasePath(other.getBasePath()),
  InsertOnly(other.getInsertOnly()),
  ModifiableMutableVerticalTable(other.getModifiableMutableVerticalTable()),
  ReturnsMutableVerticalTable(other.getReturnsMutableVerticalTable()),
  Compressed(other.getCompressed()) {
  if (other.Input != nullptr) Input = other.Input->clone();
  if (other.Header != nullptr) Header = other.Header->clone();
  if (other.ReferenceTable != nullptr) ReferenceTable = other.ReferenceTable;
}
Beispiel #9
0
void StorageManager::loadTable(std::string name, const Loader::params &parameters) {
  Loader::params *p = parameters.clone();
  p->setBasePath(Settings::getInstance()->getDBPath() + "/");
  addStorageTable(name, *p);
  delete p;
}
Beispiel #10
0
std::shared_ptr<storage::AbstractTable> RawTableLoader::load(std::shared_ptr<storage::AbstractTable> in,
        const storage::compound_metadata_list *ml,
        const Loader::params &args) {



    csv::params params;
    if (detectHeader(args.getBasePath() + _filename)) params.setLineStart(5);

    // Create the result table
    storage::metadata_vec_t v(in->columnCount());
    for(size_t i=0; i < in->columnCount(); ++i) {
        v[i] = in->metadataAt(i);
    }
    auto result = std::make_shared<storage::RawTable>(v);

    // CSV Parsing
    std::ifstream file(args.getBasePath() + _filename, std::ios::binary);
    if (!file || file.bad()) {
        throw csv::ParserError("CSV file '" + _filename + "' does not exist");
    }

    struct csv_parser parser;

    if (!csv_init(&parser, 0)) {
        csv_set_opts(&parser, CSV_APPEND_NULL);
        csv_set_delim(&parser, params.getDelimiter());

        // If there is a header in the file, we will ignore it
        std::string line;
        int line_start = params.getLineStart();

        if (line_start != 1) {
            while (line_start > 1) {
                std::getline(file, line);
                --line_start;
            }
        }

        // Prepare cb data handler
        struct raw_table_cb_data data(v);
        data.table = result;

        const size_t block_size = 16 * 1024;
        char rdbuf [block_size];

        while (file.read(rdbuf, block_size).good()) {
            auto extracted = file.gcount();
            if (extracted == 0)
                break;

            if (csv_parse(&parser,
                          rdbuf,
                          extracted,
                          (field_cb_t) raw_table_cb_per_field,
                          (line_cb_t) raw_table_cb_per_line,
                          (void*) &data) != (size_t) extracted) {
                throw csv::ParserError(csv_strerror(csv_error(&parser)));
            }
        }

        // Parse the rest
        if (csv_parse(&parser,
                      rdbuf,
                      file.gcount(),
                      (field_cb_t) raw_table_cb_per_field,
                      (line_cb_t) raw_table_cb_per_line,
                      (void*) &data) != (size_t) file.gcount()) {
            throw csv::ParserError(csv_strerror(csv_error(&parser)));
        }

        csv_fini(&parser,
                 (field_cb_t) raw_table_cb_per_field,
                 (line_cb_t) raw_table_cb_per_line,
                 (void*) &data);

    }
    csv_free(&parser);
    return result;
}