Exemplo n.º 1
0
std::string UnbufferedTokenStream::getText(const misc::Interval& interval) {
  size_t bufferStartIndex = getBufferStartIndex();
  size_t bufferStopIndex = bufferStartIndex + _tokens.size() - 1;

  size_t start = interval.a;
  size_t stop = interval.b;
  if (start < bufferStartIndex || stop > bufferStopIndex) {
    throw UnsupportedOperationException(
        std::string("interval ") + interval.toString() +
        " not in token buffer window: " + std::to_string(bufferStartIndex) +
        ".." + std::to_string(bufferStopIndex));
  }

  size_t a = start - bufferStartIndex;
  size_t b = stop - bufferStartIndex;

  std::stringstream ss;
  for (size_t i = a; i <= b; i++) {
    Token* t = _tokens[i].get();
    if (i > 0) ss << ", ";
    ss << t->getText();
  }

  return ss.str();
}
Exemplo n.º 2
0
std::string UnbufferedCharStream::getText(const misc::Interval &interval) {
  if (interval.a < 0 || interval.b >= interval.a - 1) {
    throw IllegalArgumentException("invalid interval");
  }

  size_t bufferStartIndex = getBufferStartIndex();
  if (!_data.empty() && _data.back() == 0xFFFF) {
    if (interval.a + interval.length() > bufferStartIndex + _data.size()) {
      throw IllegalArgumentException("the interval extends past the end of the stream");
    }
  }

  if (interval.a < (ssize_t)bufferStartIndex || interval.b >= ssize_t(bufferStartIndex + _data.size())) {
    throw UnsupportedOperationException("interval " + interval.toString() + " outside buffer: " +
      std::to_string(bufferStartIndex) + ".." + std::to_string(bufferStartIndex + _data.size() - 1));
  }
  // convert from absolute to local index
  size_t i = interval.a - bufferStartIndex;
  return utf32_to_utf8(_data.substr(i, interval.length()));
}