std::string LogName::canonicalize(StringPiece input) {
std::string cname;
cname.reserve(input.size());
// Ignore trailing category separator characters
size_t end = input.size();
while (end > 0 && isSeparator(input[end - 1])) {
--end;
}
bool ignoreSeparator = true;
for (size_t idx = 0; idx < end; ++idx) {
if (isSeparator(input[idx])) {
if (ignoreSeparator) {
continue;
}
cname.push_back('.');
ignoreSeparator = true;
} else {
cname.push_back(input[idx]);
ignoreSeparator = false;
}
}
return cname;
}
void parseMessage(StringPiece msg, size_t* threadID, size_t* messageIndex) {
// Validate and strip off the message prefix and suffix
constexpr StringPiece prefix{"thread "};
if (!msg.startsWith(prefix)) {
throw std::runtime_error("bad message prefix");
}
msg.advance(prefix.size());
if (!msg.endsWith(kMsgSuffix)) {
throw std::runtime_error("bad message suffix");
}
msg.subtract(kMsgSuffix.size());
// Parse then strip off the thread index
auto threadIDEnd = msg.find(' ');
if (threadIDEnd == StringPiece::npos) {
throw std::runtime_error("no middle found");
}
*threadID = folly::to<size_t>(msg.subpiece(0, threadIDEnd));
msg.advance(threadIDEnd);
// Validate that the middle of the message is what we expect,
// then strip it off
constexpr StringPiece middle{" message "};
if (!msg.startsWith(middle)) {
throw std::runtime_error("bad message middle");
}
msg.advance(middle.size());
// Parse the message index
*messageIndex = folly::to<size_t>(msg);
}
size_t qfind_first_byte_of_sse42(const StringPiece& haystack,
const StringPiece& needles) {
if (UNLIKELY(needles.empty() || haystack.empty())) {
return StringPiece::npos;
} else if (needles.size() <= 16) {
// we can save some unnecessary load instructions by optimizing for
// the common case of needles.size() <= 16
return qfind_first_byte_of_needles16(haystack, needles);
}
if (haystack.size() < 16 &&
PAGE_FOR(haystack.end() - 1) != PAGE_FOR(haystack.data() + 16)) {
// We can't safely SSE-load haystack. Use a different approach.
if (haystack.size() <= 2) {
return qfind_first_of(haystack, needles, asciiCaseSensitive);
}
return qfind_first_byte_of_byteset(haystack, needles);
}
auto ret = scanHaystackBlock<false>(haystack, needles, 0);
if (ret != StringPiece::npos) {
return ret;
}
size_t i = nextAlignedIndex(haystack.data());
for (; i < haystack.size(); i += 16) {
auto ret = scanHaystackBlock<true>(haystack, needles, i);
if (ret != StringPiece::npos) {
return ret;
}
}
return StringPiece::npos;
}
U_CAPI int32_t U_EXPORT2
unum_parseDecimal(const UNumberFormat* fmt,
const UChar* text,
int32_t textLength,
int32_t *parsePos /* 0 = start */,
char *outBuf,
int32_t outBufLength,
UErrorCode *status)
{
if (U_FAILURE(*status)) {
return -1;
}
if ((outBuf == NULL && outBufLength != 0) || outBufLength < 0) {
*status = U_ILLEGAL_ARGUMENT_ERROR;
return -1;
}
Formattable res;
parseRes(res, fmt, text, textLength, parsePos, status);
StringPiece sp = res.getDecimalNumber(*status);
if (U_FAILURE(*status)) {
return -1;
} else if (sp.size() > outBufLength) {
*status = U_BUFFER_OVERFLOW_ERROR;
} else if (sp.size() == outBufLength) {
uprv_strncpy(outBuf, sp.data(), sp.size());
*status = U_STRING_NOT_TERMINATED_WARNING;
} else {
U_ASSERT(outBufLength > 0);
uprv_strcpy(outBuf, sp.data());
}
return sp.size();
}
bool LocalSequenceFileWriter::WriteRecord(const StringPiece& key, const StringPiece& value) {
int record_size = key.size() + value.size();
int key_len = key.size();
int int_len = sizeof(int);
int size = 2 * int_len + record_size;
int used_bytes = 0;
scoped_array<char> data(new char[size]);
if (m_records_written_after_sync >= kSyncInterval) {
size += int_len + SYNC_HASH_SIZE;
data.reset(new char[size]);
used_bytes += WriteSyncToBuf(data.get() + used_bytes);
m_records_written_after_sync = 0;
}
// format:
// record_size|key_len|key|value
used_bytes += WriteInt(data.get() + used_bytes, record_size);
used_bytes += WriteInt(data.get() + used_bytes, key_len);
memcpy(data.get() + used_bytes, key.data(), key_len);
used_bytes += key_len;
memcpy(data.get() + used_bytes, value.data(), value.size());
used_bytes += value.size();
CHECK(used_bytes == size);
if (!Write(data.get(), used_bytes)) {
return false;
}
++m_records_written_after_sync;
return true;
}
void Item::resetKey(StringPiece k)
{
assert(k.size() <= 250);
keylen_ = k.size();
receivedBytes_ = 0;
append(k.data(), k.size());
hash_ = boost::hash_range(k.begin(), k.end());
}
void ReplaceAll(std::string* s, const StringPiece& from, const StringPiece& to)
{
size_t pos = 0;
while ((pos = s->find(from.data(), pos, from.size())) != std::string::npos)
{
s->replace(pos, from.size(), to.data(), to.size());
pos += to.size();
}
}
// Updates contents so that any read accesses past the last byte will
// cause a SIGSEGV. It accomplishes this by changing access to the page that
// begins immediately after the end of the contents (as allocators and mmap()
// all operate on page boundaries, this is a reasonable assumption).
// This function will also initialize buf, which caller must free().
void createProtectedBuf(StringPiece& contents, char** buf) {
ASSERT_LE(contents.size(), kPageSize);
const size_t kSuccess = 0;
char* tmp;
if (kSuccess != posix_memalign((void**)buf, kPageSize, 2 * kPageSize)) {
ASSERT_FALSE(true);
}
mprotect(*buf + kPageSize, kPageSize, PROT_NONE);
size_t newBegin = kPageSize - contents.size();
memcpy(*buf + newBegin, contents.data(), contents.size());
contents.reset(*buf + newBegin, contents.size());
}
bool IPAddressV6::validate(StringPiece ip) {
if (ip.size() > 0 && ip.front() == '[' && ip.back() == ']') {
ip = ip.subpiece(1, ip.size() - 2);
}
constexpr size_t kStrMaxLen = INET6_ADDRSTRLEN;
std::array<char, kStrMaxLen + 1> ip_cstr;
const size_t len = std::min(ip.size(), kStrMaxLen);
std::memcpy(ip_cstr.data(), ip.data(), len);
ip_cstr[len] = 0;
struct in6_addr addr;
return 1 == inet_pton(AF_INET6, ip_cstr.data(), &addr);
}
void FDSymbolizePrinter::doPrint(StringPiece sp) {
if (buffer_) {
if (sp.size() > buffer_->tailroom()) {
flush();
writeFull(fd_, sp.data(), sp.size());
} else {
memcpy(buffer_->writableTail(), sp.data(), sp.size());
buffer_->append(sp.size());
}
} else {
writeFull(fd_, sp.data(), sp.size());
}
}
FFState* TargetBigramFeature::Evaluate(const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
{
const TargetBigramState* tbState = dynamic_cast<const TargetBigramState*>(prev_state);
assert(tbState);
// current hypothesis target phrase
const Phrase& targetPhrase = cur_hypo.GetCurrTargetPhrase();
if (targetPhrase.GetSize() == 0) {
return new TargetBigramState(*tbState);
}
// extract all bigrams w1 w2 from current hypothesis
for (size_t i = 0; i < targetPhrase.GetSize(); ++i) {
const Factor* f1 = NULL;
if (i == 0) {
f1 = tbState->GetWord().GetFactor(m_factorType);
} else {
f1 = targetPhrase.GetWord(i-1).GetFactor(m_factorType);
}
const Factor* f2 = targetPhrase.GetWord(i).GetFactor(m_factorType);
const StringPiece w1 = f1->GetString();
const StringPiece w2 = f2->GetString();
// skip bigrams if they don't belong to a given restricted vocabulary
if (m_vocab.size() &&
(FindStringPiece(m_vocab, w1) == m_vocab.end() || FindStringPiece(m_vocab, w2) == m_vocab.end())) {
continue;
}
string name(w1.data(), w1.size());
name += ":";
name.append(w2.data(), w2.size());
accumulator->PlusEquals(this,name,1);
}
if (cur_hypo.GetWordsBitmap().IsComplete()) {
const StringPiece w1 = targetPhrase.GetWord(targetPhrase.GetSize()-1).GetFactor(m_factorType)->GetString();
const string& w2 = EOS_;
if (m_vocab.empty() || (FindStringPiece(m_vocab, w1) != m_vocab.end())) {
string name(w1.data(), w1.size());
name += ":";
name += w2;
accumulator->PlusEquals(this,name,1);
}
return NULL;
}
return new TargetBigramState(targetPhrase.GetWord(targetPhrase.GetSize()-1));
}
int RML_RE::TryMatch(const StringPiece& text,
int startpos,
Anchor anchor,
int *vec,
int vecsize) const {
pcre* re = (anchor == ANCHOR_BOTH) ? re_full_ : re_partial_;
if (re == NULL) {
//fprintf(stderr, "Matching against invalid re: %s\n", error_->c_str());
return 0;
}
pcre_extra extra = { 0 };
if (options_.match_limit() > 0) {
extra.flags = PCRE_EXTRA_MATCH_LIMIT;
extra.match_limit = options_.match_limit();
}
int rc = pcre_exec(re, // The regular expression object
&extra,
text.data(),
text.size(),
startpos,
(anchor == UNANCHORED) ? 0 : PCRE_ANCHORED,
vec,
vecsize);
// Handle errors
if (rc == PCRE_ERROR_NOMATCH) {
return 0;
} else if (rc < 0) {
//fprintf(stderr, "Unexpected return code: %d when matching '%s'\n",
// re, pattern_.c_str());
return 0;
} else if (rc == 0) {
// pcre_exec() returns 0 as a special case when the number of
// capturing subpatterns exceeds the size of the vector.
// When this happens, there is a match and the output vector
// is filled, but we miss out on the positions of the extra subpatterns.
rc = vecsize / 2;
}
if ((anchor == ANCHOR_BOTH) && (re_full_ == re_partial_)) {
// We need an extra check to make sure that the match extended
// to the end of the input string
assert(vec[0] == 0); // PCRE_ANCHORED forces starting match
if (vec[1] != text.size()) return 0; // Did not get ending match
}
return rc;
}
size_t qfind_first_byte_of_memchr(const StringPiece& haystack,
const StringPiece& needles) {
size_t best = haystack.size();
for (char needle: needles) {
const void* ptr = memchr(haystack.data(), needle, best);
if (ptr) {
auto found = static_cast<const char*>(ptr) - haystack.data();
best = std::min<size_t>(best, found);
}
}
if (best == haystack.size()) {
return StringPiece::npos;
}
return best;
}
bool Base64::WebSafeEncode(const StringPiece& input, std::string* output)
{
std::string temp;
temp.resize(modp_b64r_encode_len(input.size())); // makes room for null byte
// null terminates result since result is base64 text!
int input_size = static_cast<int>(input.size());
int output_size = modp_b64r_encode(&(temp[0]), input.data(), input_size);
if (output_size < 0)
return false;
temp.resize(output_size); // strips off null byte
output->swap(temp);
return true;
}
bool Base64::WebSafeDecode(const StringPiece& input, std::string* output)
{
std::string temp;
temp.resize(modp_b64r_decode_len(input.size()));
// does not null terminate result since result is binary data!
int input_size = static_cast<int>(input.size());
int output_size = modp_b64r_decode(&(temp[0]), input.data(), input_size);
if (output_size < 0)
return false;
temp.resize(output_size);
output->swap(temp);
return true;
}
// Updates contents so that any read accesses past the last byte will
// cause a SIGSEGV. It accomplishes this by changing access to the page that
// begins immediately after the end of the contents (as allocators and mmap()
// all operate on page boundaries, this is a reasonable assumption).
// This function will also initialize buf, which caller must free().
void createProtectedBuf(StringPiece& contents, char** buf) {
ASSERT_LE(contents.size(), kPageSize);
const size_t kSuccess = 0;
char* pageAlignedBuf = (char*)aligned_malloc(2 * kPageSize, kPageSize);
if (pageAlignedBuf == nullptr) {
ASSERT_FALSE(true);
}
// Protect the page after the first full page-aligned region of the
// malloc'ed buffer
mprotect(pageAlignedBuf + kPageSize, kPageSize, PROT_NONE);
size_t newBegin = kPageSize - contents.size();
memcpy(pageAlignedBuf + newBegin, contents.data(), contents.size());
contents.reset(pageAlignedBuf + newBegin, contents.size());
*buf = pageAlignedBuf;
}
void Session::doDelete(Session::Tokenizer::iterator& beg, Session::Tokenizer::iterator end)
{
assert(command_ == "delete");
// FIXME: check (beg != end)
StringPiece key = *beg;
bool good = key.size() <= kLongestKeySize;
++beg;
if (!good)
{
reply("CLIENT_ERROR bad command line format\r\n");
}
else if (beg != end && *beg != "0") // issue 108, old protocol
{
reply("CLIENT_ERROR bad command line format. Usage: delete <key> [noreply]\r\n");
}
else
{
needle_->resetKey(key);
if (owner_->deleteItem(needle_))
{
reply("DELETED\r\n");
}
else
{
reply("NOT_FOUND\r\n");
}
}
}
void Session::reply(StringPiece msg)
{
if (!noreply_)
{
conn_->send(msg.data(), msg.size());
}
}
StringPiece LogName::getParent(StringPiece name) {
if (name.empty()) {
return name;
}
ssize_t idx = name.size();
// Skip over any trailing separator characters
while (idx > 0 && isSeparator(name[idx - 1])) {
--idx;
}
// Now walk backwards to the next separator character
while (idx > 0 && !isSeparator(name[idx - 1])) {
--idx;
}
// And again skip over any separator characters, in case there are multiple
// repeated characters.
while (idx > 0 && isSeparator(name[idx - 1])) {
--idx;
}
return StringPiece(name.begin(), idx);
}
void DoSplitLines(
const StringPiece& full,
std::vector<StringType>* result,
bool keep_line_endling
)
{
result->clear();
size_t prev_pos = 0;
size_t pos;
StringType token;
while ((pos = full.find('\n', prev_pos)) != std::string::npos)
{
token.assign(full.data() + prev_pos, pos - prev_pos + 1);
if (!keep_line_endling)
RemoveLineEnding(&token);
result->push_back(token);
prev_pos = pos + 1;
}
if (prev_pos < full.size())
{
token.assign(full.data() + prev_pos, full.length() - prev_pos);
if (!keep_line_endling)
RemoveLineEnding(&token);
result->push_back(token);
}
}
std::shared_ptr<ElfFile> SignalSafeElfCache::getFile(StringPiece p) {
if (p.size() > Path::kMaxSize) {
return nullptr;
}
Path path(p);
auto pos = map_.find(path);
if (pos != map_.end()) {
return slots_[pos->second];
}
size_t n = map_.size();
if (n >= slots_.size()) {
DCHECK_EQ(map_.size(), slots_.size());
return nullptr;
}
auto& f = slots_[n];
const char* msg = "";
int r = f->openNoThrow(path.data(), true, &msg);
if (r != ElfFile::kSuccess) {
return nullptr;
}
map_[path] = n;
return f;
}
size_t LogName::hash(StringPiece name) {
// Code based on StringPiece::hash(), but which ignores leading and trailing
// category separator characters, as well as multiple consecutive separator
// characters, so equivalent names result in the same hash.
uint32_t hash = 5381;
size_t end = name.size();
while (end > 0 && isSeparator(name[end - 1])) {
--end;
}
bool ignoreSeparator = true;
for (size_t idx = 0; idx < end; ++idx) {
uint8_t value;
if (isSeparator(name[idx])) {
if (ignoreSeparator) {
continue;
}
value = '.';
ignoreSeparator = true;
} else {
value = static_cast<uint8_t>(name[idx]);
ignoreSeparator = false;
}
hash = ((hash << 5) + hash) + value;
}
return hash;
}
bool RML_RE::Rewrite(string *out, const StringPiece &rewrite,
const StringPiece &text, int *vec, int veclen) const {
for (const char *s = rewrite.data(), *end = s + rewrite.size();
s < end; s++) {
int c = *s;
if (c == '\\') {
c = *++s;
if (isdigit(c)) {
int n = (c - '0');
if (n >= veclen) {
//fprintf(stderr, requested group %d in regexp %.*s\n",
// n, rewrite.size(), rewrite.data());
return false;
}
int start = vec[2 * n];
if (start >= 0)
out->append(text.data() + start, vec[2 * n + 1] - start);
} else if (c == '\\') {
out->push_back('\\');
} else {
//fprintf(stderr, "invalid rewrite pattern: %.*s\n",
// rewrite.size(), rewrite.data());
return false;
}
} else {
out->push_back(c);
}
}
return true;
}
bool HttpMessage::ParseHeaders(const StringPiece& data, HttpMessage::ErrorCode* error) {
HttpMessage::ErrorCode error_placeholder;
if (error == NULL)
error = &error_placeholder;
StringPiece::size_type pos = data.find_first_of('\n');
if (pos == StringPiece::npos) {
pos = data.size();
}
std::string first_line =
StringTrimRight(data.substr(0, pos), "\r");
if (first_line.empty()) {
*error = HttpMessage::ERROR_NO_START_LINE;
return false;
}
if (!ParseStartLine(first_line, error))
return false;
int error_code = 0;
bool result = m_headers.Parse(data.substr(pos + 1), &error_code);
*error = static_cast<HttpMessage::ErrorCode>(error_code);
return result;
}
void expect(LineReader& lr, const char* expected) {
StringPiece line;
size_t expectedLen = strlen(expected);
EXPECT_EQ(expectedLen != 0 ? LineReader::kReading : LineReader::kEof,
lr.readLine(line));
EXPECT_EQ(expectedLen, line.size());
EXPECT_EQ(std::string(expected, expectedLen), line.str());
}
U_EXPORT UBool U_EXPORT2
operator==(const StringPiece& x, const StringPiece& y) {
int32_t len = x.size();
if (len != y.size()) {
return false;
}
if (len == 0) {
return true;
}
const char* p = x.data();
const char* p2 = y.data();
// Test last byte in case strings share large common prefix
--len;
if (p[len] != p2[len]) return false;
// At this point we can, but don't have to, ignore the last byte.
return uprv_memcmp(p, p2, len) == 0;
}
std::string writeTempFile(const StringPiece prefix, const StringPiece content) {
char buf[512];
snprintf(buf, sizeof buf, "/tmp/%s-runScript-%ld-%d-%05d-XXXXXX",
prefix.data(),
now(),
::getpid(),
++g_tempFileCount);
int tempfd = ::mkstemp(buf);
if(!setFd(tempfd, FD_CLOEXEC)){
LOG_ERROR << "Set FD_CLOEXEC failed.";
}
ssize_t n = ::pwrite(tempfd, content.data(), content.size(), 0);
::fchmod(tempfd, 0755);
::close(tempfd);
return n == content.size() ? buf : "";
}
/**
* Set the DigitList from a decimal number string.
*
* The incoming string _must_ be nul terminated, even though it is arriving
* as a StringPiece because that is what the decNumber library wants.
* We can get away with this for an internal function; it would not
* be acceptable for a public API.
*/
void
DigitList::set(StringPiece source, UErrorCode &status, uint32_t /*fastpathBits*/) {
if (U_FAILURE(status)) {
return;
}
#if 0
if(fastpathBits==(kFastpathOk|kNoDecimal)) {
int32_t size = source.size();
const char *data = source.data();
int64_t r = 0;
int64_t m = 1;
// fast parse
while(size>0) {
char ch = data[--size];
if(ch=='+') {
break;
} else if(ch=='-') {
r = -r;
break;
} else {
int64_t d = ch-'0';
//printf("CH[%d]=%c, %d, *=%d\n", size,ch, (int)d, (int)m);
r+=(d)*m;
m *= 10;
}
}
//printf("R=%d\n", r);
set(r);
} else
#endif
{
// Figure out a max number of digits to use during the conversion, and
// resize the number up if necessary.
int32_t numDigits = source.length();
if (numDigits > fContext.digits) {
// fContext.digits == fStorage.getCapacity()
decNumber *t = fStorage.resize(numDigits, fStorage.getCapacity());
if (t == NULL) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
fDecNumber = t;
fContext.digits = numDigits;
}
fContext.status = 0;
uprv_decNumberFromString(fDecNumber, source.data(), &fContext);
if ((fContext.status & DEC_Conversion_syntax) != 0) {
status = U_DECIMAL_NUMBER_SYNTAX_ERROR;
}
}
internalClear();
}
TEST(StringPiece, Constexpr) {
constexpr const char* helloArray = "hello";
constexpr StringPiece hello1("hello");
EXPECT_EQ("hello", hello1);
static_assert(hello1.size() == 5, "hello size should be 5 at compile time");
constexpr StringPiece hello2(helloArray);
EXPECT_EQ("hello", hello2);
static_assert(hello2.size() == 5, "hello size should be 5 at compile time");
}
// Replace the first "old" pattern with the "new" pattern in a string
std::string ReplaceFirst(
const StringPiece& s,
const StringPiece& oldsub,
const StringPiece& newsub)
{
if (oldsub.empty())
return s.as_string();
std::string res;
std::string::size_type pos = s.find(oldsub);
if (pos == std::string::npos)
return s.as_string();
else
{
res.append(s.data(), pos);
res.append(newsub.data(), newsub.size());
res.append(s.data() + pos + oldsub.size(), s.length() - pos - oldsub.size());
}
return res;
}