Ejemplo n.º 1
0
std::string LogName::canonicalize(StringPiece input) {
  std::string cname;
  cname.reserve(input.size());

  // Ignore trailing category separator characters
  size_t end = input.size();
  while (end > 0 && isSeparator(input[end - 1])) {
    --end;
  }

  bool ignoreSeparator = true;
  for (size_t idx = 0; idx < end; ++idx) {
    if (isSeparator(input[idx])) {
      if (ignoreSeparator) {
        continue;
      }
      cname.push_back('.');
      ignoreSeparator = true;
    } else {
      cname.push_back(input[idx]);
      ignoreSeparator = false;
    }
  }
  return cname;
}
Ejemplo n.º 2
0
  void parseMessage(StringPiece msg, size_t* threadID, size_t* messageIndex) {
    // Validate and strip off the message prefix and suffix
    constexpr StringPiece prefix{"thread "};
    if (!msg.startsWith(prefix)) {
      throw std::runtime_error("bad message prefix");
    }
    msg.advance(prefix.size());
    if (!msg.endsWith(kMsgSuffix)) {
      throw std::runtime_error("bad message suffix");
    }
    msg.subtract(kMsgSuffix.size());

    // Parse then strip off the thread index
    auto threadIDEnd = msg.find(' ');
    if (threadIDEnd == StringPiece::npos) {
      throw std::runtime_error("no middle found");
    }
    *threadID = folly::to<size_t>(msg.subpiece(0, threadIDEnd));
    msg.advance(threadIDEnd);

    // Validate that the middle of the message is what we expect,
    // then strip it off
    constexpr StringPiece middle{" message "};
    if (!msg.startsWith(middle)) {
      throw std::runtime_error("bad message middle");
    }
    msg.advance(middle.size());

    // Parse the message index
    *messageIndex = folly::to<size_t>(msg);
  }
Ejemplo n.º 3
0
size_t qfind_first_byte_of_sse42(const StringPiece& haystack,
                                 const StringPiece& needles) {
  if (UNLIKELY(needles.empty() || haystack.empty())) {
    return StringPiece::npos;
  } else if (needles.size() <= 16) {
    // we can save some unnecessary load instructions by optimizing for
    // the common case of needles.size() <= 16
    return qfind_first_byte_of_needles16(haystack, needles);
  }

  if (haystack.size() < 16 &&
      PAGE_FOR(haystack.end() - 1) != PAGE_FOR(haystack.data() + 16)) {
    // We can't safely SSE-load haystack. Use a different approach.
    if (haystack.size() <= 2) {
      return qfind_first_of(haystack, needles, asciiCaseSensitive);
    }
    return qfind_first_byte_of_byteset(haystack, needles);
  }

  auto ret = scanHaystackBlock<false>(haystack, needles, 0);
  if (ret != StringPiece::npos) {
    return ret;
  }

  size_t i = nextAlignedIndex(haystack.data());
  for (; i < haystack.size(); i += 16) {
    auto ret = scanHaystackBlock<true>(haystack, needles, i);
    if (ret != StringPiece::npos) {
      return ret;
    }
  }

  return StringPiece::npos;
}
Ejemplo n.º 4
0
U_CAPI int32_t U_EXPORT2
unum_parseDecimal(const UNumberFormat*  fmt,
            const UChar*    text,
            int32_t         textLength,
            int32_t         *parsePos /* 0 = start */,
            char            *outBuf,
            int32_t         outBufLength,
            UErrorCode      *status)
{
    if (U_FAILURE(*status)) {
        return -1;
    }
    if ((outBuf == NULL && outBufLength != 0) || outBufLength < 0) {
        *status = U_ILLEGAL_ARGUMENT_ERROR;
        return -1;
    }
    Formattable res;
    parseRes(res, fmt, text, textLength, parsePos, status);
    StringPiece sp = res.getDecimalNumber(*status);
    if (U_FAILURE(*status)) {
       return -1;
    } else if (sp.size() > outBufLength) {
        *status = U_BUFFER_OVERFLOW_ERROR;
    } else if (sp.size() == outBufLength) {
        uprv_strncpy(outBuf, sp.data(), sp.size());
        *status = U_STRING_NOT_TERMINATED_WARNING;
    } else {
        U_ASSERT(outBufLength > 0);
        uprv_strcpy(outBuf, sp.data());
    }
    return sp.size();
}
Ejemplo n.º 5
0
bool LocalSequenceFileWriter::WriteRecord(const StringPiece& key, const StringPiece& value) {
    int record_size = key.size() + value.size();
    int key_len = key.size();
    int int_len = sizeof(int);
    int size = 2 * int_len + record_size;
    int used_bytes = 0;
    scoped_array<char> data(new char[size]);

    if (m_records_written_after_sync >= kSyncInterval) {
        size += int_len + SYNC_HASH_SIZE;
        data.reset(new char[size]);
        used_bytes += WriteSyncToBuf(data.get() + used_bytes);
        m_records_written_after_sync = 0;
    }

    // format:
    // record_size|key_len|key|value

    used_bytes += WriteInt(data.get() + used_bytes, record_size);
    used_bytes += WriteInt(data.get() + used_bytes, key_len);

    memcpy(data.get() + used_bytes, key.data(), key_len);
    used_bytes += key_len;
    memcpy(data.get() + used_bytes, value.data(), value.size());
    used_bytes += value.size();

    CHECK(used_bytes == size);

    if (!Write(data.get(), used_bytes)) {
        return false;
    }

    ++m_records_written_after_sync;
    return true;
}
Ejemplo n.º 6
0
void Item::resetKey(StringPiece k)
{
	assert(k.size() <= 250);
	keylen_ = k.size();
	receivedBytes_ = 0;
	append(k.data(), k.size());
	hash_ = boost::hash_range(k.begin(), k.end());
}
Ejemplo n.º 7
0
void ReplaceAll(std::string* s, const StringPiece& from, const StringPiece& to)
{
    size_t pos = 0;
    while ((pos = s->find(from.data(), pos, from.size())) != std::string::npos)
    {
        s->replace(pos, from.size(), to.data(), to.size());
        pos += to.size();
    }
}
Ejemplo n.º 8
0
// Updates contents so that any read accesses past the last byte will
// cause a SIGSEGV.  It accomplishes this by changing access to the page that
// begins immediately after the end of the contents (as allocators and mmap()
// all operate on page boundaries, this is a reasonable assumption).
// This function will also initialize buf, which caller must free().
void createProtectedBuf(StringPiece& contents, char** buf) {
  ASSERT_LE(contents.size(), kPageSize);
  const size_t kSuccess = 0;
  char* tmp;
  if (kSuccess != posix_memalign((void**)buf, kPageSize, 2 * kPageSize)) {
    ASSERT_FALSE(true);
  }
  mprotect(*buf + kPageSize, kPageSize, PROT_NONE);
  size_t newBegin = kPageSize - contents.size();
  memcpy(*buf + newBegin, contents.data(), contents.size());
  contents.reset(*buf + newBegin, contents.size());
}
Ejemplo n.º 9
0
bool IPAddressV6::validate(StringPiece ip) {
    if (ip.size() > 0 && ip.front() == '[' && ip.back() == ']') {
        ip = ip.subpiece(1, ip.size() - 2);
    }

    constexpr size_t kStrMaxLen = INET6_ADDRSTRLEN;
    std::array<char, kStrMaxLen + 1> ip_cstr;
    const size_t len = std::min(ip.size(), kStrMaxLen);
    std::memcpy(ip_cstr.data(), ip.data(), len);
    ip_cstr[len] = 0;
    struct in6_addr addr;
    return 1 == inet_pton(AF_INET6, ip_cstr.data(), &addr);
}
Ejemplo n.º 10
0
void FDSymbolizePrinter::doPrint(StringPiece sp) {
  if (buffer_) {
    if (sp.size() > buffer_->tailroom()) {
      flush();
      writeFull(fd_, sp.data(), sp.size());
    } else {
      memcpy(buffer_->writableTail(), sp.data(), sp.size());
      buffer_->append(sp.size());
    }
  } else {
    writeFull(fd_, sp.data(), sp.size());
  }
}
Ejemplo n.º 11
0
FFState* TargetBigramFeature::Evaluate(const Hypothesis& cur_hypo,
                                       const FFState* prev_state,
                                       ScoreComponentCollection* accumulator) const
{
  const TargetBigramState* tbState = dynamic_cast<const TargetBigramState*>(prev_state);
  assert(tbState);

  // current hypothesis target phrase
  const Phrase& targetPhrase = cur_hypo.GetCurrTargetPhrase();
  if (targetPhrase.GetSize() == 0) {
    return new TargetBigramState(*tbState);
  }

  // extract all bigrams w1 w2 from current hypothesis
  for (size_t i = 0; i < targetPhrase.GetSize(); ++i) {
    const Factor* f1 = NULL;
    if (i == 0) {
      f1 = tbState->GetWord().GetFactor(m_factorType);
    } else {
      f1 = targetPhrase.GetWord(i-1).GetFactor(m_factorType);
    }
    const Factor* f2 = targetPhrase.GetWord(i).GetFactor(m_factorType);
    const StringPiece w1 = f1->GetString();
    const StringPiece w2 = f2->GetString();

    // skip bigrams if they don't belong to a given restricted vocabulary
    if (m_vocab.size() &&
        (FindStringPiece(m_vocab, w1) == m_vocab.end() || FindStringPiece(m_vocab, w2) == m_vocab.end())) {
      continue;
    }

    string name(w1.data(), w1.size());
    name += ":";
    name.append(w2.data(), w2.size());
    accumulator->PlusEquals(this,name,1);
  }

  if (cur_hypo.GetWordsBitmap().IsComplete()) {
    const StringPiece w1 = targetPhrase.GetWord(targetPhrase.GetSize()-1).GetFactor(m_factorType)->GetString();
    const string& w2 = EOS_;
    if (m_vocab.empty() || (FindStringPiece(m_vocab, w1) != m_vocab.end())) {
      string name(w1.data(), w1.size());
      name += ":";
      name += w2;
      accumulator->PlusEquals(this,name,1);
    }
    return NULL;
  }
  return new TargetBigramState(targetPhrase.GetWord(targetPhrase.GetSize()-1));
}
Ejemplo n.º 12
0
int RML_RE::TryMatch(const StringPiece& text,
                 int startpos,
                 Anchor anchor,
                 int *vec,
                 int vecsize) const {
  pcre* re = (anchor == ANCHOR_BOTH) ? re_full_ : re_partial_;
  if (re == NULL) {
    //fprintf(stderr, "Matching against invalid re: %s\n", error_->c_str());
    return 0;
  }

  pcre_extra extra = { 0 };
  if (options_.match_limit() > 0) {
    extra.flags = PCRE_EXTRA_MATCH_LIMIT;
    extra.match_limit = options_.match_limit();
  }
  int rc = pcre_exec(re,              // The regular expression object
                     &extra,
                     text.data(),
                     text.size(),
                     startpos,
                     (anchor == UNANCHORED) ? 0 : PCRE_ANCHORED,
                     vec,
                     vecsize);

  // Handle errors
  if (rc == PCRE_ERROR_NOMATCH) {
    return 0;
  } else if (rc < 0) {
    //fprintf(stderr, "Unexpected return code: %d when matching '%s'\n",
    //        re, pattern_.c_str());
    return 0;
  } else if (rc == 0) {
    // pcre_exec() returns 0 as a special case when the number of
    // capturing subpatterns exceeds the size of the vector.
    // When this happens, there is a match and the output vector
    // is filled, but we miss out on the positions of the extra subpatterns.
    rc = vecsize / 2;
  }

  if ((anchor == ANCHOR_BOTH) && (re_full_ == re_partial_)) {
    // We need an extra check to make sure that the match extended
    // to the end of the input string
    assert(vec[0] == 0);                 // PCRE_ANCHORED forces starting match
    if (vec[1] != text.size()) return 0; // Did not get ending match
  }

  return rc;
}
Ejemplo n.º 13
0
size_t qfind_first_byte_of_memchr(const StringPiece& haystack,
                                  const StringPiece& needles) {
  size_t best = haystack.size();
  for (char needle: needles) {
    const void* ptr = memchr(haystack.data(), needle, best);
    if (ptr) {
      auto found = static_cast<const char*>(ptr) - haystack.data();
      best = std::min<size_t>(best, found);
    }
  }
  if (best == haystack.size()) {
    return StringPiece::npos;
  }
  return best;
}
Ejemplo n.º 14
0
bool Base64::WebSafeEncode(const StringPiece& input, std::string* output)
{
    std::string temp;
    temp.resize(modp_b64r_encode_len(input.size()));  // makes room for null byte

    // null terminates result since result is base64 text!
    int input_size = static_cast<int>(input.size());
    int output_size = modp_b64r_encode(&(temp[0]), input.data(), input_size);
    if (output_size < 0)
        return false;

    temp.resize(output_size);  // strips off null byte
    output->swap(temp);
    return true;
}
Ejemplo n.º 15
0
bool Base64::WebSafeDecode(const StringPiece& input, std::string* output)
{
    std::string temp;
    temp.resize(modp_b64r_decode_len(input.size()));

    // does not null terminate result since result is binary data!
    int input_size = static_cast<int>(input.size());
    int output_size = modp_b64r_decode(&(temp[0]), input.data(), input_size);
    if (output_size < 0)
        return false;

    temp.resize(output_size);
    output->swap(temp);
    return true;
}
Ejemplo n.º 16
0
// Updates contents so that any read accesses past the last byte will
// cause a SIGSEGV.  It accomplishes this by changing access to the page that
// begins immediately after the end of the contents (as allocators and mmap()
// all operate on page boundaries, this is a reasonable assumption).
// This function will also initialize buf, which caller must free().
void createProtectedBuf(StringPiece& contents, char** buf) {
  ASSERT_LE(contents.size(), kPageSize);
  const size_t kSuccess = 0;
  char* pageAlignedBuf = (char*)aligned_malloc(2 * kPageSize, kPageSize);
  if (pageAlignedBuf == nullptr) {
    ASSERT_FALSE(true);
  }
  // Protect the page after the first full page-aligned region of the
  // malloc'ed buffer
  mprotect(pageAlignedBuf + kPageSize, kPageSize, PROT_NONE);
  size_t newBegin = kPageSize - contents.size();
  memcpy(pageAlignedBuf + newBegin, contents.data(), contents.size());
  contents.reset(pageAlignedBuf + newBegin, contents.size());
  *buf = pageAlignedBuf;
}
Ejemplo n.º 17
0
void Session::doDelete(Session::Tokenizer::iterator& beg, Session::Tokenizer::iterator end)
{
	assert(command_ == "delete");
	// FIXME: check (beg != end)
	StringPiece key = *beg;
	bool good = key.size() <= kLongestKeySize;
	++beg;
	if (!good)
	{
		reply("CLIENT_ERROR bad command line format\r\n");
	}
	else if (beg != end && *beg != "0") // issue 108, old protocol
	{
		reply("CLIENT_ERROR bad command line format.  Usage: delete <key> [noreply]\r\n");
	}
	else
	{
		needle_->resetKey(key);
		if (owner_->deleteItem(needle_))
		{
			reply("DELETED\r\n");
		}
		else
		{
			reply("NOT_FOUND\r\n");
		}
	}
}
Ejemplo n.º 18
0
void Session::reply(StringPiece msg)
{
	if (!noreply_)
	{
		conn_->send(msg.data(), msg.size());
	}
}
Ejemplo n.º 19
0
StringPiece LogName::getParent(StringPiece name) {
  if (name.empty()) {
    return name;
  }

  ssize_t idx = name.size();

  // Skip over any trailing separator characters
  while (idx > 0 && isSeparator(name[idx - 1])) {
    --idx;
  }

  // Now walk backwards to the next separator character
  while (idx > 0 && !isSeparator(name[idx - 1])) {
    --idx;
  }

  // And again skip over any separator characters, in case there are multiple
  // repeated characters.
  while (idx > 0 && isSeparator(name[idx - 1])) {
    --idx;
  }

  return StringPiece(name.begin(), idx);
}
Ejemplo n.º 20
0
void DoSplitLines(
    const StringPiece& full,
    std::vector<StringType>* result,
    bool keep_line_endling
)
{
    result->clear();
    size_t prev_pos = 0;
    size_t pos;
    StringType token;
    while ((pos = full.find('\n', prev_pos)) != std::string::npos)
    {
        token.assign(full.data() + prev_pos, pos - prev_pos + 1);
        if (!keep_line_endling)
            RemoveLineEnding(&token);
        result->push_back(token);
        prev_pos = pos + 1;
    }
    if (prev_pos < full.size())
    {
        token.assign(full.data() + prev_pos, full.length() - prev_pos);
        if (!keep_line_endling)
            RemoveLineEnding(&token);
        result->push_back(token);
    }
}
Ejemplo n.º 21
0
std::shared_ptr<ElfFile> SignalSafeElfCache::getFile(StringPiece p) {
  if (p.size() > Path::kMaxSize) {
    return nullptr;
  }

  Path path(p);
  auto pos = map_.find(path);
  if (pos != map_.end()) {
    return slots_[pos->second];
  }

  size_t n = map_.size();
  if (n >= slots_.size()) {
    DCHECK_EQ(map_.size(), slots_.size());
    return nullptr;
  }

  auto& f = slots_[n];

  const char* msg = "";
  int r = f->openNoThrow(path.data(), true, &msg);
  if (r != ElfFile::kSuccess) {
    return nullptr;
  }

  map_[path] = n;
  return f;
}
Ejemplo n.º 22
0
size_t LogName::hash(StringPiece name) {
  // Code based on StringPiece::hash(), but which ignores leading and trailing
  // category separator characters, as well as multiple consecutive separator
  // characters, so equivalent names result in the same hash.
  uint32_t hash = 5381;

  size_t end = name.size();
  while (end > 0 && isSeparator(name[end - 1])) {
    --end;
  }

  bool ignoreSeparator = true;
  for (size_t idx = 0; idx < end; ++idx) {
    uint8_t value;
    if (isSeparator(name[idx])) {
      if (ignoreSeparator) {
        continue;
      }
      value = '.';
      ignoreSeparator = true;
    } else {
      value = static_cast<uint8_t>(name[idx]);
      ignoreSeparator = false;
    }
    hash = ((hash << 5) + hash) + value;
  }
  return hash;
}
Ejemplo n.º 23
0
bool RML_RE::Rewrite(string *out, const StringPiece &rewrite,
                 const StringPiece &text, int *vec, int veclen) const {
  for (const char *s = rewrite.data(), *end = s + rewrite.size();
       s < end; s++) {
    int c = *s;
    if (c == '\\') {
      c = *++s;
      if (isdigit(c)) {
        int n = (c - '0');
        if (n >= veclen) {
          //fprintf(stderr, requested group %d in regexp %.*s\n",
          //        n, rewrite.size(), rewrite.data());
          return false;
        }
        int start = vec[2 * n];
        if (start >= 0)
          out->append(text.data() + start, vec[2 * n + 1] - start);
      } else if (c == '\\') {
        out->push_back('\\');
      } else {
        //fprintf(stderr, "invalid rewrite pattern: %.*s\n",
        //        rewrite.size(), rewrite.data());
        return false;
      }
    } else {
      out->push_back(c);
    }
  }
  return true;
}
Ejemplo n.º 24
0
bool HttpMessage::ParseHeaders(const StringPiece& data, HttpMessage::ErrorCode* error) {
    HttpMessage::ErrorCode error_placeholder;
    if (error == NULL)
        error = &error_placeholder;

    StringPiece::size_type pos = data.find_first_of('\n');
    if (pos == StringPiece::npos) {
        pos = data.size();
    }
    std::string first_line =
        StringTrimRight(data.substr(0, pos), "\r");

    if (first_line.empty()) {
        *error = HttpMessage::ERROR_NO_START_LINE;
        return false;
    }

    if (!ParseStartLine(first_line, error))
        return false;

    int error_code = 0;
    bool result = m_headers.Parse(data.substr(pos + 1), &error_code);
    *error = static_cast<HttpMessage::ErrorCode>(error_code);
    return result;
}
Ejemplo n.º 25
0
void expect(LineReader& lr, const char* expected) {
  StringPiece line;
  size_t expectedLen = strlen(expected);
  EXPECT_EQ(expectedLen != 0 ? LineReader::kReading : LineReader::kEof,
            lr.readLine(line));
  EXPECT_EQ(expectedLen, line.size());
  EXPECT_EQ(std::string(expected, expectedLen), line.str());
}
Ejemplo n.º 26
0
U_EXPORT UBool U_EXPORT2
operator==(const StringPiece& x, const StringPiece& y) {
  int32_t len = x.size();
  if (len != y.size()) {
    return false;
  }
  if (len == 0) {
    return true;
  }
  const char* p = x.data();
  const char* p2 = y.data();
  // Test last byte in case strings share large common prefix
  --len;
  if (p[len] != p2[len]) return false;
  // At this point we can, but don't have to, ignore the last byte.
  return uprv_memcmp(p, p2, len) == 0;
}
Ejemplo n.º 27
0
std::string writeTempFile(const StringPiece prefix, const StringPiece content) {
    char buf[512];
    snprintf(buf, sizeof buf, "/tmp/%s-runScript-%ld-%d-%05d-XXXXXX",
                      prefix.data(),
                      now(),
                      ::getpid(),
                      ++g_tempFileCount);

    int tempfd = ::mkstemp(buf);
    if(!setFd(tempfd, FD_CLOEXEC)){
    	LOG_ERROR << "Set FD_CLOEXEC failed.";
    }
    ssize_t n = ::pwrite(tempfd, content.data(), content.size(), 0);
    ::fchmod(tempfd, 0755);
    ::close(tempfd);

    return n == content.size() ? buf : "";
}
Ejemplo n.º 28
0
/**
 * Set the DigitList from a decimal number string.
 *
 * The incoming string _must_ be nul terminated, even though it is arriving
 * as a StringPiece because that is what the decNumber library wants.
 * We can get away with this for an internal function; it would not
 * be acceptable for a public API.
 */
void
DigitList::set(StringPiece source, UErrorCode &status, uint32_t /*fastpathBits*/) {
    if (U_FAILURE(status)) {
        return;
    }

#if 0
    if(fastpathBits==(kFastpathOk|kNoDecimal)) {
      int32_t size = source.size();
      const char *data = source.data();
      int64_t r = 0;
      int64_t m = 1;
      // fast parse
      while(size>0) {
        char ch = data[--size];
        if(ch=='+') {
          break;
        } else if(ch=='-') {
          r = -r;
          break;
        } else {
          int64_t d = ch-'0';
          //printf("CH[%d]=%c, %d, *=%d\n", size,ch, (int)d, (int)m);
          r+=(d)*m;
          m *= 10;
        }
      }
      //printf("R=%d\n", r);
      set(r);
    } else
#endif
        {
      // Figure out a max number of digits to use during the conversion, and
      // resize the number up if necessary.
      int32_t numDigits = source.length();
      if (numDigits > fContext.digits) {
        // fContext.digits == fStorage.getCapacity()
        decNumber *t = fStorage.resize(numDigits, fStorage.getCapacity());
        if (t == NULL) {
          status = U_MEMORY_ALLOCATION_ERROR;
          return;
        }
        fDecNumber = t;
        fContext.digits = numDigits;
      }

      fContext.status = 0;
      uprv_decNumberFromString(fDecNumber, source.data(), &fContext);
      if ((fContext.status & DEC_Conversion_syntax) != 0) {
        status = U_DECIMAL_NUMBER_SYNTAX_ERROR;
      }
    }
    internalClear();
}
Ejemplo n.º 29
0
TEST(StringPiece, Constexpr) {
  constexpr const char* helloArray = "hello";

  constexpr StringPiece hello1("hello");
  EXPECT_EQ("hello", hello1);
  static_assert(hello1.size() == 5, "hello size should be 5 at compile time");

  constexpr StringPiece hello2(helloArray);
  EXPECT_EQ("hello", hello2);
  static_assert(hello2.size() == 5, "hello size should be 5 at compile time");
}
Ejemplo n.º 30
0
// Replace the first "old" pattern with the "new" pattern in a string
std::string ReplaceFirst(
    const StringPiece& s,
    const StringPiece& oldsub,
    const StringPiece& newsub)
{
    if (oldsub.empty())
        return s.as_string();

    std::string res;
    std::string::size_type pos = s.find(oldsub);
    if (pos == std::string::npos)
        return s.as_string();
    else
    {
        res.append(s.data(), pos);
        res.append(newsub.data(), newsub.size());
        res.append(s.data() + pos + oldsub.size(), s.length() - pos - oldsub.size());
    }
    return res;
}