예제 #1
0
s3_device::s3_device(const std::string& filename, const bool write) {
  m_filename = filename;
  // split out the access key and secret key
  webstor::s3url url;
  webstor::parse_s3url(filename, url);
  m_s3fs = std::make_shared<dmlc::io::S3FileSystem>();
  m_s3fs->SetCredentials(url.access_key_id, url.secret_key);
  std::string url_without_credentials;
  if (url.endpoint.empty()) {
     url_without_credentials = "s3://" + url.bucket + "/" + url.object_name;
  } else {
    url_without_credentials = "s3://" + url.endpoint + "/" + url.bucket + "/" + url.object_name;
  }
  auto uri = dmlc::io::URI(url_without_credentials.c_str());
  if (write) {
    m_write_stream.reset(m_s3fs->Open(uri, "w"));
  } else {
    try {
      auto pathinfo = m_s3fs->GetPathInfo(uri);
      m_filesize = pathinfo.size;
      if (pathinfo.type != dmlc::io::kFile) {
        log_and_throw("Cannot open " + sanitize_url(filename));
      }
      m_read_stream.reset(m_s3fs->OpenForRead(uri));
    } catch (...) {
      log_and_throw("Cannot open " + sanitize_url(filename));
    }
  }
}
예제 #2
0
 void unity_global::save_model(std::shared_ptr<model_base> model,
                               const std::string& model_wrapper,
                               const std::string& url) {
   logstream(LOG_INFO) << "Save model to " << sanitize_url(url) << std::endl;
   logstream(LOG_INFO) << "Model name: " << model->name() << std::endl;
   try {
     dir_archive dir;
     dir.open_directory_for_write(url);
     dir.set_metadata("contents", "model");
     oarchive oarc(dir);
     oarc.write(CLASS_MAGIC_HEADER, strlen(CLASS_MAGIC_HEADER));
     oarc << model->name();
     oarc << model_wrapper;
     oarc << *model;
     if (dir.get_output_stream()->fail()) {
       std::string message = "Fail to write.";
       log_and_throw_io_failure(message);
     }
     dir.close();
   } catch (std::ios_base::failure& e) {
     std::string message = "Unable to save model to " + sanitize_url(url) + ": " + e.what();
     log_and_throw_io_failure(message);
   } catch (std::string& e) {
     log_and_throw(std::string("Unable to save model to ") + sanitize_url(url) + ": " + e);
   } catch (...) {
     log_and_throw(std::string("Unknown Error: Unable to save model to ") + sanitize_url(url));
   }
 }
예제 #3
0
bool unity_sgraph::save_graph(std::string target, std::string format) {
  log_func_entry();
  try {
    if (format == "binary") {
      dir_archive dir;
      dir.open_directory_for_write(target);
      dir.set_metadata("contents", "graph");
      oarchive oarc(dir);
      if (dir.get_output_stream()->fail()) {
        log_and_throw_io_failure("Fail to write");
      }
      save(oarc);
      dir.close();
    } else if (format == "json") {
      save_sgraph_to_json(get_graph(), target);
    } else if (format == "csv") {
      save_sgraph_to_csv(get_graph(), target);
    } else {
      log_and_throw("Unable to save to format : " + format);
    }
  } catch (std::ios_base::failure& e) {
    std::string message =
        "Unable to save graph to " + sanitize_url(target) + ": " + e.what();
    log_and_throw_io_failure(message);
  } catch (std::string& e) {
    std::string message =
        "Unable to save graph to " + sanitize_url(target) + ": " + e;
    log_and_throw(message);
  } catch (...) {
    std::string message =
        "Unable to save graph to " + sanitize_url(target) + ": Unknown Error.";
    log_and_throw(message);
  }
  return true;
}
예제 #4
0
void unity_sgraph::fast_validate_add_edges(const sframe& edges,
    std::string src_field,
    std::string dst_field,
    size_t groupa, size_t groupb) const {

  if (!edges.contains_column(src_field)) {
    log_and_throw("Input sframe does not contain source id column: " + src_field);
  }
  if (!edges.contains_column(dst_field)) {
    log_and_throw("Input sframe does not contain target id column: " + dst_field);
  }

  flex_type_enum src_id_type = edges.column_type(edges.column_index(src_field));
  flex_type_enum dst_id_type = edges.column_type(edges.column_index(dst_field));

  if (src_id_type != dst_id_type) {
    std::string msg = "Source and target ids have different types: ";
    msg += std::string(flex_type_enum_to_name(src_id_type)) + " != " + flex_type_enum_to_name(dst_id_type);
    log_and_throw(msg);
  }

  if (src_id_type != flex_type_enum::INTEGER && src_id_type != flex_type_enum::STRING) {
    log_and_throw(
        std::string("Invalid id column type : ")
        + flex_type_enum_to_name(src_id_type)
        + ". Supported types are: integer and string."
    );
  }
}
예제 #5
0
bool unity_sgraph::load_graph(std::string target_dir) {
  log_func_entry();
  try {
    dir_archive dir;
    dir.open_directory_for_read(target_dir);
    std::string contents;
    if (dir.get_metadata("contents", contents) == false ||
        contents != "graph") {
      log_and_throw(std::string("Archive does not contain a graph."));
    }
    iarchive iarc(dir);
    load(iarc);
    dir.close();
  } catch (std::ios_base::failure& e) {
    std::string message = "Unable to load graph from " + sanitize_url(target_dir)
      + ": " + e.what();
    log_and_throw_io_failure(message);
  } catch (std::string& e) {
    std::string message = "Unable to load graph from " + sanitize_url(target_dir)
      + ": " + e;
    log_and_throw(message);
  } catch (...) {
    std::string message = "Unable to load graph from " + sanitize_url(target_dir)
      + ": Unknown Error.";
    log_and_throw(message);
  }
  return true;
}
예제 #6
0
std::shared_ptr<unity_sgraph_base>
unity_sgraph::lambda_triple_apply(const std::string& lambda_str,
                                  const std::vector<std::string>& mutated_fields) {
  log_func_entry();
  if (mutated_fields.empty()) {
    log_and_throw("mutated_fields cannot be empty");
  }
  std::shared_ptr<sgraph> g = std::make_shared<sgraph>((*m_graph)());
  std::vector<std::string> mutated_vertex_fields, mutated_edge_fields;
  const auto& all_vertex_fields = g->get_vertex_fields();
  const auto& all_edge_fields = g->get_edge_fields();
  std::set<std::string> all_vertex_field_set(all_vertex_fields.begin(), all_vertex_fields.end());
  std::set<std::string> all_edge_field_set(all_edge_fields.begin(), all_edge_fields.end());
  for (auto& f : mutated_fields) {
    if (f == sgraph::VID_COLUMN_NAME || f == sgraph::SRC_COLUMN_NAME || f == sgraph::DST_COLUMN_NAME) {
      log_and_throw("mutated fields cannot contain id field: " + f);
    }
    if (!all_vertex_field_set.count(f) && !all_edge_field_set.count(f)) {
      log_and_throw("mutated field \"" + f + "\" cannot be found in graph");
    }
    if (all_vertex_field_set.count(f))
      mutated_vertex_fields.push_back(f);
    if (all_edge_field_set.count(f))
      mutated_edge_fields.push_back(f);
  }
  DASSERT_FALSE(mutated_fields.empty());
  sgraph_compute::triple_apply(*g, lambda_str, mutated_vertex_fields, mutated_edge_fields);
  std::shared_ptr<unity_sgraph> ret(new unity_sgraph(g));
  return ret;
}
예제 #7
0
std::vector<std::pair<flexible_type,gl_sframe>>
grouped_sframe::iterator_get_next(size_t len) {
  if(!m_inited)
    log_and_throw("The 'group' operation needs to occur before iteration!");
  if(!m_iterating)
    log_and_throw("Must begin iteration before iterating!");

  std::vector<std::pair<flexible_type,gl_sframe>> ret;

  if(len < 1) {
    return ret;
  } else {
    auto items_left = m_range_directory.size() - m_cur_iterator_idx;
    if(len > items_left) {
      len = items_left;
    }
    ret.resize(len);
  }

  size_t cur_cnt = 0;
  for(; (m_cur_iterator_idx < m_range_directory.size()) && (cur_cnt < len);
      ++m_cur_iterator_idx, ++cur_cnt) {
    auto sf = this->get_group_by_index(m_cur_iterator_idx);
    auto name = m_group_names[m_cur_iterator_idx];
    ret[cur_cnt] = std::make_pair(name, sf);
  }

  if(cur_cnt < len) {
    m_iterating = false;
  }

  return ret;
}
예제 #8
0
void process::close_read_pipe() {
  if(!m_launched)
    log_and_throw("No process launched!");
  if(!m_launched_with_popen || m_read_handle == NULL)
    log_and_throw("Cannot close read pipe from child, no pipe initialized.");
  CloseHandle(m_read_handle);
  m_read_handle = NULL;
}
예제 #9
0
sframe join(sframe& sf_left, 
            sframe& sf_right,
            std::string join_type,
            const std::map<std::string,std::string> join_columns,
            size_t max_buffer_size) {
  // ***SANITY CHECKS 

  // check that each sframe is valid
  if(!sf_left.num_rows() || !sf_left.num_columns()) {
    log_and_throw("Current SFrame has nothing to join!");
  }

  if(!sf_right.num_rows() || !sf_right.num_columns()) {
    log_and_throw("Given SFrame has nothing to join!");
  }

  std::vector<size_t> left_join_positions;
  std::vector<size_t> right_join_positions;
  for(const auto &col_pair : join_columns) {
    // Check that all columns exist (in both sframes)
    // These will throw if not found
    left_join_positions.push_back(sf_left.column_index(col_pair.first));
    right_join_positions.push_back(sf_right.column_index(col_pair.second));
    
    // Each column must have matching types to compare effectively
    if(sf_left.column_type(left_join_positions.back()) !=
        sf_right.column_type(right_join_positions.back())) {
      log_and_throw("Columns " + col_pair.first + " and " + col_pair.second + 
          " does not have the same type in both SFrames.");
    }
  }
  
  // Figure out what join type we have to do
  boost::algorithm::to_lower(join_type);

  join_type_t in_join_type;
  if(join_type == "outer") {
    in_join_type = FULL_JOIN;
  } else if(join_type == "left") {
    in_join_type = LEFT_JOIN;
  } else if(join_type == "right") {
    in_join_type = RIGHT_JOIN;
  } else if(join_type == "inner") {
    in_join_type = INNER_JOIN;
  } else {
    log_and_throw("Invalid join type given!");
  }

  // execute join (perhaps multiplex algorithm based on something?)
  join_impl::hash_join_executor join_executor(sf_left,
                                              sf_right,
                                              left_join_positions,
                                              right_join_positions,
                                              in_join_type,
                                              max_buffer_size);

  return join_executor.grace_hash_join();
}
예제 #10
0
 /** 
  * Return the begin iterator of the segment.
  * The iterator (\ref sarray_iterator) is of the input iterator type and 
  * has value_type T. See \ref end() to get the end iterator of the segment.
  *
  * The iterator is invalid once the originating sarray is destroyed.
  * Accessing the iterator after the sarray is destroyed is undefined behavior.
  *
  * \code
  * // example to print segment 1 to screen
  * auto iter = sarr.begin(1);
  * auto enditer =sarr.end(1);
  * while(iter != enditer) {
  *   std::cout << *iter << "\n";
  *   ++iter;
  * }
  * \endcode
  *
  * Will throw an exception if the sarray is invalid (there is an error
  * reading files) Also segmentid must be a valid segment ID. Will throw an
  * exception otherwise.
  */
 iterator begin(size_t segmentid) const {
   std::lock_guard<mutex> lck(lock);
   if (opened_segments.count(segmentid) == 0) {
     opened_segments.insert(segmentid);
   } else {
     log_and_throw(std::string("Must reset sarray iterators!"));
   }
   if (reader == NULL) log_and_throw(std::string("Invalid sarray"));
   if (segmentid >= num_segments()) log_and_throw(std::string("Invalid segment ID"));
   return iterator(&(m_read_buffers[segmentid]), segmentid, true);
 }
예제 #11
0
/**
 * Parse the png image info from in memory buffer.
 */
void parse_png(const char* data, size_t length,
               size_t& width, size_t& height,
               size_t& channels) {

  // Begin of setup
  png_structp png_ptr;
  png_infop info_ptr;
  setup_png_reader(data, length, &png_ptr, &info_ptr);

  // Construct the simple in memory buffer 
  png_memory_buffer source;
  source.data = (char*)data;
  source.length = length;
  source.offset = 0;

  // Set custom read function
  png_set_read_fn(png_ptr, (png_voidp)&source, (png_rw_ptr)png_memread_func);
  png_read_info(png_ptr, info_ptr);
  // End of setup

  png_uint_32 _width = 0;
  png_uint_32 _height = 0;
  int bit_depth = 0;
  int color_type = -1;
  png_uint_32 retval = png_get_IHDR(png_ptr, info_ptr,
      &_width,
      &_height,
      &bit_depth,
      &color_type,
      NULL, NULL, NULL);

  if(retval != 1) {
    logstream(LOG_ERROR) << "Fail parsing PNG header" << std::endl;
    png_destroy_read_struct(&png_ptr, &info_ptr, NULL);
    throw(std::string("Invalid PNG file"));
  }

  width = _width;
  height = _height;
  if (bit_depth != 8) {
    png_destroy_read_struct(&png_ptr, &info_ptr, NULL);
    log_and_throw(std::string("Unsupported PNG bit depth: " + std::to_string(bit_depth)));
  }

  channels = png_num_channels(color_type);
  if (channels != 1 && channels != 3 && channels != 4) {
    png_destroy_read_struct(&png_ptr, &info_ptr, NULL);
    log_and_throw(std::string("Unsupported PNG color type: ") + std::to_string(color_type));
  }

  png_destroy_read_struct(&png_ptr, &info_ptr, NULL);
}
예제 #12
0
continuous_bins continuous_result::get_bins(flex_int num_bins) const {
  if (num_bins < 1) {
    log_and_throw("num_bins must be positive.");
  }
  continuous_bins ret;

  // determine the bin range that covers min to max
  size_t first_bin = get_bin_idx(min, scale_min, scale_max);
  size_t last_bin = get_bin_idx(max, scale_min, scale_max);
  size_t effective_bins = (last_bin - first_bin) + 1;

  // Might end up with fewer effective bins due to very small
  // number of unique values. For now, comment out this assert.
  // TODO -- what should we assert here instead, to make sure we have enough
  // effective range for the desired number of bins? Or should we force
  // discrete histogram for very low cardinality? (At which point, we keep
  // this assertion).
  //DASSERT_GE(effective_bins, (MAX_BINS/4));

  if (num_bins > (MAX_BINS/4)) {
    log_and_throw("num_bins must be less than or equal to the effective number of bins available.");
  }

  // rescale to desired bins, taking more than the effective range if
  // necessary in order to get to num_bins total without resampling
  size_t bins_per_bin = effective_bins / num_bins;
  size_t overflow = effective_bins % num_bins;
  size_t before = 0;
  size_t after = 0;
  if (overflow) {
    overflow = num_bins - overflow;
    bins_per_bin = (effective_bins + overflow) / num_bins;
    before = overflow / 2;
    after = (overflow / 2) + (overflow % 2);
  }

  ret.bins = flex_list(num_bins, 0); // initialize empty
  ret.min = get_value_at_bin(std::max<ssize_t>(0, first_bin - before), scale_min, scale_max, MAX_BINS);
  ret.max = get_value_at_bin(std::min<ssize_t>(last_bin + after + 1, MAX_BINS), scale_min, scale_max, MAX_BINS);
  for (size_t i=0; i<num_bins; i++) {
    for (size_t j=0; j<bins_per_bin; j++) {
      ssize_t idx = (i * bins_per_bin) + j + (first_bin - before);
      if (idx < 0 || idx >= MAX_BINS) {
        // don't try to get values below 0, or past MAX_BINS, that would be silly
        continue;
      }
      ret.bins[i] += this->bins[idx];
    }
  }
  return ret;
}
예제 #13
0
void unity_sgraph::fast_validate_add_vertices(const sframe& vertices,
    std::string id_field, size_t group) const {
  if (!vertices.contains_column(id_field)) {
    log_and_throw("Input sframe does not contain id column: " + id_field);
  }
  flex_type_enum id_type = vertices.column_type(vertices.column_index(id_field));

  if (id_type != flex_type_enum::INTEGER && id_type != flex_type_enum::STRING) {
    log_and_throw(
        std::string("Invalid id column type : ")
        + flex_type_enum_to_name(id_type) 
        + ". Supported types are: integer and string."
    );
  }
}
예제 #14
0
variant_type simple_model::get_value(std::string key, variant_map_type& opts) {
  if (params.count(key)) {
    return params[key];
  } else {
    log_and_throw("Key " + key + " not found in model.");
  }
}
std::vector<T> read_sequence_section(const boost::property_tree::ptree& data, 
                                     std::string key,
                                     size_t expected_elements) {
  std::vector<T> ret;
  if (expected_elements == 0) return ret;
  const boost::property_tree::ptree& section = data.get_child(key);
  ret.resize(expected_elements);

  // loop through the children of the column_names section
  size_t sid = 0;
  for(const auto& val: section) {
    const auto& key = val.first;
    if (key.empty()) {
      // this is the array-like sequences
      ret[sid] = boost::lexical_cast<T>(val.second.get_value<std::string>());
      ++sid;
    } else {
      // this is a dictionary-like sequence
      sid = std::stoi(key);
      if (sid >= ret.size()) {
        log_and_throw(std::string("Invalid ID in ") + key + " section."
                      "Segment IDs are expected to be sequential.");
      }
      ret[sid] = boost::lexical_cast<T>(val.second.get_value<std::string>());
    }
  }
  return ret;
}
예제 #16
0
jpeg_error_exit (j_common_ptr cinfo)
{
  /* Always display the message. */
  /* We could postpone this until after returning, if we chose. */
  (*cinfo->err->output_message) (cinfo);
  log_and_throw(std::string("Unexpected JPEG decode failure"));
}
예제 #17
0
 /**
  * Reads the value of a key associated with the sarray.
  * Returns true on success, false on failure.
  */
 bool get_metadata(std::string key, std::string &val) const {
   ASSERT_NE(reader, NULL);
   if (reader == NULL) log_and_throw(std::string("Invalid sarray"));
   bool ret = false;
   std::tie(ret, val) = get_metadata(key);
   return ret;
 }
예제 #18
0
void continuous::init(const gl_sarray& source) {
  continuous_parent::init(source);
  flex_type_enum dtype = m_source.dtype();
  if (dtype != flex_type_enum::INTEGER &&
      dtype != flex_type_enum::FLOAT) {
    log_and_throw("dtype of the provided SArray is not valid for histogram. Only int and float are valid dtypes.");
  }

  size_t input_size = m_source.size();
  if (input_size >= 2 &&
      m_source[0].get_type() != flex_type_enum::UNDEFINED &&
      m_source[1].get_type() != flex_type_enum::UNDEFINED) {
    // start with a sane range for the bins (somewhere near the data)
    // (it can be exceptionally small, since the doubling used in resize()
    // will make it converge to the real range quickly)
    m_transformer.init(m_source[0], m_source[1]);
  } else if (input_size == 1 &&
             m_source[0].get_type() != flex_type_enum::UNDEFINED) {
    // one value, not so interesting
    m_transformer.init(m_source[0], m_source[0]);
  } else {
    // no data
    m_transformer.init(0.0, 0.0);
  }
}
예제 #19
0
파일: io.cpp 프로젝트: Hannah1999/Dato-Core
void read_raw_image(const std::string& url, char** data, size_t& length, size_t& width, size_t& height, size_t& channels, Format& format, const std::string& format_hint) {

  general_ifstream fin(url);
  length = fin.file_size();
  *data = new char[length];
  try {
    fin.read(*data, length);
    if (format_hint == "JPG") {
      format = Format::JPG;
    } else if (format_hint == "PNG") {
      format = Format::PNG;
    } else {
      if (boost::algorithm::iends_with(url, "jpg") ||
          boost::algorithm::iends_with(url, "jpeg")) {
        format = Format::JPG;
      } else if (boost::algorithm::iends_with(url, "png")) {
        format = Format::PNG;
      }
    }
    if (format == Format::JPG) {
      parse_jpeg(*data, length, width, height, channels);
    } else if (format == Format::PNG) {
      parse_png(*data, length, width, height, channels);
    } else {
      log_and_throw(std::string("Unsupported image format. Supported formats are JPG and PNG"));
    }
  } catch (...) {
    delete[] *data;
    *data = NULL;
    length = 0;
    throw;
  }
  fin.close();
};
예제 #20
0
 static void call_lua_function(lua::Value& function, const flexible_type& arg, flexible_type& ret) {
   lua::Value valret;
   switch(arg.get_type()) {
    case flex_type_enum::INTEGER:
      valret = function(arg.get<flex_int>());
      break;
    case flex_type_enum::FLOAT:
      valret = function(arg.get<flex_float>());
      break;
    case flex_type_enum::STRING:
      valret = function(arg.get<flex_string>().c_str());
      break;
    default:
      log_and_throw("Not Supported at the moment");
   };
   if (valret.is<lua::Integer>()) {
     lua::Integer val = 0;
     valret.get<lua::Integer>(val);
     ret = val;
   } else if (valret.is<lua::Number>()) {
     lua::Number val = 0;
     valret.get<lua::Number>(val);
     ret = val;
   } else if (valret.is<lua::String>()) {
     std::string val;
     valret.get<std::string>(val);
     ret = std::move(val);
   } else {
     ret = FLEX_UNDEFINED;
   }
 }
예제 #21
0
 std::streamsize read(char* strm_ptr, std::streamsize n) {
   // there is an upper limit of how many bytes we can read
   // based on the file size
   n = std::min<std::streamsize>(n, m_file_size - m_file_pos);
   std::streamsize ret = 0;
   while(n > 0) {
     // the block number containing the offset.
     auto block_number = m_file_pos / READ_CACHING_BLOCK_SIZE;
     // the offset inside the block
     auto block_offset = m_file_pos % READ_CACHING_BLOCK_SIZE;
     // number of bytes I can read inside this block before I hit the next block
     size_t n_bytes = (block_number + 1) * READ_CACHING_BLOCK_SIZE - m_file_pos;
     n_bytes = std::min<size_t>(n_bytes, n);
     bool success = fetch_block(strm_ptr + ret,
                                block_number,
                                block_offset,
                                n_bytes);
     if (success == false) {
       log_and_throw(std::string("Unable to read ") + m_filename);
     }
     n -= n_bytes;
     ret += n_bytes;
     // advance the file position
     m_file_pos += n_bytes;
   }
   return ret;
 }
예제 #22
0
  flex_dict_view::flex_dict_view(const flexible_type& value) {
    if (value.get_type() == flex_type_enum::DICT) {
      m_flex_dict_ptr = &(value.get<flex_dict>());
      return;
    }

    log_and_throw("Cannot construct a flex_dict_view object from type ");
  }
예제 #23
0
 std::map<std::string, flexible_type> unity_global::describe_toolkit_function(std::string name) {
   auto spec = toolkit_functions->get_toolkit_function_info(name);
   if (spec == NULL) {
     log_and_throw(std::string("No such toolkit!"));
   } else {
     return spec->description;
   }
 }
예제 #24
0
  variant_map_type unity_global::load_model(const std::string& url) {
    logstream(LOG_INFO) << "Load model from " << sanitize_url(url) << std::endl;
    try {

      dir_archive dir;
      dir.open_directory_for_read(url);
      std::string contents;
      if (dir.get_metadata("contents", contents) == false || contents != "model") {
        log_and_throw(std::string("Archive does not contain a model."));
      }
      iarchive iarc(dir);

      std::string model_name;
      std::string model_wrapper;
      char buf[256] = "";
      size_t magic_header_size = strlen(CLASS_MAGIC_HEADER);
      iarc.read(buf, magic_header_size);
      if (strcmp(buf, CLASS_MAGIC_HEADER)) {
        log_and_throw(std::string("Invalid model file."));
      }
      iarc >> model_name;
      logstream(LOG_INFO) << "Model name: " << model_name << std::endl;
      iarc >> model_wrapper;
      std::shared_ptr<model_base> model_ptr = classes->get_toolkit_class(model_name);
      iarc  >> *(model_ptr);
      if (dir.get_input_stream()->fail()) {
        std::string message = "Fail to read.";
        log_and_throw_io_failure(message);
      }
      dir.close();
      variant_map_type ret;
      variant_set_value<std::shared_ptr<model_base>>(ret["model_base"], model_ptr);
      flexible_type flex_model_wrapper = (flexible_type)model_wrapper;
      variant_set_value<flexible_type>(ret["model_wrapper"], flex_model_wrapper);
      return ret;
    } catch (std::ios_base::failure& e) {
      std::string message = "Unable to load model from " + sanitize_url(url) + ": " + e.what();
      log_and_throw_io_failure(message);
    } catch (std::string& e) {
      log_and_throw(std::string("Unable to load model from ") + sanitize_url(url) + ": " + e);
    } catch (const std::exception& e) {
      log_and_throw(std::string("Unable to load model from ") + sanitize_url(url) + ": " + e.what());
    } catch (...) {
      log_and_throw(std::string("Unknown Error: Unable to load model from ") + sanitize_url(url));
    }
  }
std::map<std::string, flexible_type> 
toolkit_class_registry::get_toolkit_class_description(const std::string& class_name) {
  if (descriptions.count(class_name)) {
    return descriptions[class_name];
  } else {
    log_and_throw(std::string("Class" + class_name + " does not exist."));
  }
}
예제 #26
0
T safe_varmap_get(const variant_map_type& kv, std::string key) {
  if (kv.count(key) == 0) {
    log_and_throw("Required Key " + key + " not found");
  } else {
    return variant_get_value<T>(kv.at(key));
  }
  __builtin_unreachable();
}
std::shared_ptr<model_base> toolkit_class_registry::get_toolkit_class(
    const std::string& class_name) {
  if (registry.count(class_name)) {
    return std::shared_ptr<model_base>(registry[class_name]());
  } else {
    log_and_throw(std::string("Class " + class_name + " does not exist."));
  }
}
예제 #28
0
/**
 * Utility function to throw an error if a vector is of unequal length.
 * \param[in] gl_sarray of type vector 
 */
void check_vector_equal_size(const gl_sarray& in) {
  // Initialize. 
  DASSERT_TRUE(in.dtype() == flex_type_enum::VECTOR); 
  size_t n_threads = thread::cpu_count();
  n_threads = std::max(n_threads, size_t(1));
  size_t m_size = in.size();
          
  // Throw the following error. 
  auto throw_error = [] (size_t row_number, size_t expected, size_t current) {
    std::stringstream ss;
    ss << "Vectors must be of the same size. Row " << row_number 
       << " contains a vector of size " << current << ". Expected a vector of"
       << " size " << expected << "." << std::endl;
    log_and_throw(ss.str());
  };
  
  // Within each block of the SArray, check that the vectors have the same size.
  std::vector<size_t> expected_sizes (n_threads, size_t(-1));
  in_parallel([&](size_t thread_idx, size_t n_threads) {
    size_t start_row = thread_idx * m_size / n_threads; 
    size_t end_row = (thread_idx + 1) * m_size / n_threads;
    size_t expected_size = size_t(-1);
    size_t row_number = start_row;
    for (const auto& v: in.range_iterator(start_row, end_row)) {
      if (v != FLEX_UNDEFINED) {
        if (expected_size == size_t(-1)) {
          expected_size = v.size();
          expected_sizes[thread_idx] = expected_size; 
        } else {
          DASSERT_TRUE(v.get_type() == flex_type_enum::VECTOR);
          if (expected_size != v.size()) {
            throw_error(row_number, expected_size, v.size());
          }
        }
      }
      row_number++;
    }
  });

  // Make sure sizes accross blocks are also the same. 
  size_t vector_size = size_t(-1);
  for (size_t thread_idx = 0; thread_idx < n_threads; thread_idx++) {
    // If this block contains all None values, skip it.
    if (expected_sizes[thread_idx] != size_t(-1)) {

      if (vector_size == size_t(-1)) {
          vector_size = expected_sizes[thread_idx]; 
      } else {
         if (expected_sizes[thread_idx] != vector_size) {
           throw_error(thread_idx * m_size / n_threads, 
                              vector_size, expected_sizes[thread_idx]);
         } 
      }
    }
  }
}
예제 #29
0
 static bool file_contains_substring(std::string file,
                                     std::string substring) {
   general_ifstream fin(file);
   if (fin.fail()) {
     log_and_throw("Cannot open " + file);
   }
   size_t fsize = fin.file_size();
   if (fsize == (size_t)(-1)) {
     log_and_throw("Cannot open " + file);
   }
   char* buf = new char[fsize];
   fin.read(buf, fsize);
   auto f = boost::algorithm::boyer_moore_search(buf, buf + fsize, 
                                                 substring.begin(), substring.end());
   // return is true if found
   bool ret = (f != buf + fsize);
   delete [] buf;
   return ret;
 }
예제 #30
0
파일: malloc.c 프로젝트: jajm/libgends
void * gds_realloc(void *ptr, size_t size)
{
    ptr = realloc(ptr, size);
    if (ptr == NULL) {
        log_and_throw(NotEnoughMemoryException,
                      "realloc failed to allocate %d bytes", size);
    }

    return ptr;
}