void McRequest::Keys::update(folly::StringPiece key) {
keyWithoutRoute = key;
if (!key.empty()) {
if (*key.begin() == '/') {
size_t pos = 1;
for (int i = 0; i < 2; ++i) {
pos = key.find('/', pos);
if (pos == std::string::npos) {
break;
}
++pos;
}
if (pos != std::string::npos) {
keyWithoutRoute.advance(pos);
routingPrefix.reset(key.begin(), pos);
}
}
}
routingKey = keyWithoutRoute;
size_t pos = keyWithoutRoute.find("|#|");
if (pos != std::string::npos) {
routingKey.reset(keyWithoutRoute.begin(), pos);
}
routingKeyHash = getMemcacheKeyHashValue(routingKey);
}
std::string MessagePrinter::serializeMessageHeader(
folly::StringPiece messageName,
mc_res_t result,
const std::string& key) {
std::string out;
if (options_.script) {
out.append(folly::sformat("\"type\": \"{}\"", messageName.data()));
if (result != mc_res_unknown) {
out.append(
folly::sformat(",\n \"result\": \"{}\"", mc_res_to_string(result)));
}
if (!key.empty()) {
out.append(
folly::sformat(",\n \"key\": \"{}\"", folly::backslashify(key)));
}
} else {
out.append(messageName.data());
if (result != mc_res_unknown) {
out.push_back(' ');
out.append(mc_res_to_string(result));
}
if (!key.empty()) {
out.push_back(' ');
out.append(folly::backslashify(key));
}
}
return out;
}
void Keys::update(folly::StringPiece key) {
keyWithoutRoute_ = key;
if (!key.empty()) {
if (*key.begin() == '/') {
size_t pos = 1;
for (int i = 0; i < 2; ++i) {
pos = key.find('/', pos);
if (pos == std::string::npos) {
break;
}
++pos;
}
if (pos != std::string::npos) {
keyWithoutRoute_.advance(pos);
routingPrefix_.reset(key.begin(), pos);
}
}
}
routingKey_ = keyWithoutRoute_;
size_t pos = keyWithoutRoute_.find("|#|");
if (pos != std::string::npos) {
routingKey_.reset(keyWithoutRoute_.begin(), pos);
}
routingKeyHash_ = 0;
}
// Skip over padding until sp.data() - start is a multiple of alignment
void skipPadding(folly::StringPiece& sp, const char* start, size_t alignment) {
size_t remainder = (sp.data() - start) % alignment;
if (remainder) {
FOLLY_SAFE_CHECK(alignment - remainder <= sp.size(), "invalid padding");
sp.advance(alignment - remainder);
}
}
size_t WeightedCh3HashFunc::operator()(folly::StringPiece key) const {
auto n = weights_.size();
checkLogic(n && n <= furc_maximum_pool_size(), "Invalid pool size: {}", n);
size_t salt = 0;
size_t index = 0;
std::string saltedKey;
auto originalKey = key;
for (size_t i = 0; i < kNumTries; ++i) {
index = furc_hash(key.data(), key.size(), n);
/* Use 32-bit hash, but store in 64-bit ints so that
we don't have to deal with overflows */
uint64_t p = folly::hash::SpookyHashV2::Hash32(key.data(), key.size(),
kHashSeed);
assert(0 <= weights_[index] && weights_[index] <= 1.0);
uint64_t w = weights_[index] * std::numeric_limits<uint32_t>::max();
/* Rehash only if p is out of range */
if (LIKELY(p < w)) {
return index;
}
/* Change the key to rehash */
auto s = salt++;
saltedKey = originalKey.str();
do {
saltedKey.push_back(char(s % 10) + '0');
s /= 10;
} while (s > 0);
key = saltedKey;
}
return index;
}
void SSLContext::loadCertificateFromBufferPEM(folly::StringPiece cert) {
if (cert.data() == nullptr) {
throw std::invalid_argument("loadCertificate: <cert> is nullptr");
}
ssl::BioUniquePtr bio(BIO_new(BIO_s_mem()));
if (bio == nullptr) {
throw std::runtime_error("BIO_new: " + getErrors());
}
int written = BIO_write(bio.get(), cert.data(), cert.size());
if (written <= 0 || static_cast<unsigned>(written) != cert.size()) {
throw std::runtime_error("BIO_write: " + getErrors());
}
ssl::X509UniquePtr x509(
PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr));
if (x509 == nullptr) {
throw std::runtime_error("PEM_read_bio_X509: " + getErrors());
}
if (SSL_CTX_use_certificate(ctx_, x509.get()) == 0) {
throw std::runtime_error("SSL_CTX_use_certificate: " + getErrors());
}
}
void SSLContext::loadPrivateKeyFromBufferPEM(folly::StringPiece pkey) {
if (pkey.data() == nullptr) {
throw std::invalid_argument("loadPrivateKey: <pkey> is nullptr");
}
ssl::BioUniquePtr bio(BIO_new(BIO_s_mem()));
if (bio == nullptr) {
throw std::runtime_error("BIO_new: " + getErrors());
}
int written = BIO_write(bio.get(), pkey.data(), pkey.size());
if (written <= 0 || static_cast<unsigned>(written) != pkey.size()) {
throw std::runtime_error("BIO_write: " + getErrors());
}
ssl::EvpPkeyUniquePtr key(
PEM_read_bio_PrivateKey(bio.get(), nullptr, nullptr, nullptr));
if (key == nullptr) {
throw std::runtime_error("PEM_read_bio_PrivateKey: " + getErrors());
}
if (SSL_CTX_use_PrivateKey(ctx_, key.get()) == 0) {
throw std::runtime_error("SSL_CTX_use_PrivateKey: " + getErrors());
}
}
/** Adds an asynchronous request to the event log. */
void asynclog_delete(proxy_t* proxy,
const ProxyClientCommon& pclient,
folly::StringPiece key,
folly::StringPiece poolName) {
dynamic json = {};
const auto& host = pclient.ap->getHost();
const auto& port = pclient.ap->getPort();
if (proxy->opts.use_asynclog_version2) {
json = dynamic::object;
json["f"] = proxy->opts.flavor_name;
json["h"] = folly::sformat("[{}]:{}", host, port);
json["p"] = poolName.str();
json["k"] = key.str();
} else {
/* ["host", port, escaped_command] */
json.push_back(host);
json.push_back(port);
json.push_back(folly::sformat("delete {}\r\n", key));
}
auto fd = asynclog_open(proxy);
if (!fd) {
logFailure(proxy->router, memcache::failure::Category::kSystemError,
"asynclog_open() failed (key {}, pool {})",
key, poolName);
return;
}
// ["AS1.0", 1289416829.836, "C", ["10.0.0.1", 11302, "delete foo\r\n"]]
// OR ["AS2.0", 1289416829.836, "C", {"f":"flavor","h":"[10.0.0.1]:11302",
// "p":"pool_name","k":"foo\r\n"}]
dynamic jsonOut = {};
if (proxy->opts.use_asynclog_version2) {
jsonOut.push_back(ASYNCLOG_MAGIC2);
} else {
jsonOut.push_back(ASYNCLOG_MAGIC);
}
struct timeval timestamp;
CHECK(gettimeofday(×tamp, nullptr) == 0);
auto timestamp_ms =
facebook::memcache::to<std::chrono::milliseconds>(timestamp).count();
jsonOut.push_back(1e-3 * timestamp_ms);
jsonOut.push_back(std::string("C"));
jsonOut.push_back(json);
auto jstr = folly::toJson(jsonOut) + "\n";
ssize_t size = folly::writeFull(fd->fd(), jstr.data(), jstr.size());
if (size == -1 || size_t(size) < jstr.size()) {
logFailure(proxy->router, memcache::failure::Category::kSystemError,
"Error fully writing asynclog request (key {}, pool {})",
key, poolName);
}
}
typename std::enable_if<std::is_pod<T>::value, T>::type
read(folly::StringPiece& sp) {
FOLLY_SAFE_CHECK(sp.size() >= sizeof(T), "underflow");
T x;
memcpy(&x, sp.data(), sizeof(T));
sp.advance(sizeof(T));
return x;
}
// Read a null-terminated string
folly::StringPiece readNullTerminated(folly::StringPiece& sp) {
const char* p = static_cast<const char*>(
memchr(sp.data(), 0, sp.size()));
FOLLY_SAFE_CHECK(p, "invalid null-terminated string");
folly::StringPiece ret(sp.data(), p);
sp.assign(p + 1, sp.end());
return ret;
}
void operator ()(
T const &,
T const &,
folly::StringPiece path,
folly::StringPiece
) const {
out_.emplace_back(path.data(), path.size());
}
bool HTTPHeaders::exists(folly::StringPiece name) const {
const HTTPHeaderCode code = HTTPCommonHeaders::hash(name.data(),
name.size());
if (code != HTTP_HEADER_OTHER) {
return exists(code);
} else {
ITERATE_OVER_STRINGS(name, { return true; });
return false;
}
TEST(FixedStringConversionTest, ConversionToFollyRange) {
// The following declaraction is static for compilers that haven't implemented
// the resolution of:
// http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#1454
static constexpr folly::FixedString<16> tmp{"This is a string"};
constexpr folly::StringPiece piece = tmp;
static_assert(tmp.begin() == piece.begin(), "");
static_assert(tmp.end() == piece.end(), "");
}
void SwSwitch::initThread(folly::StringPiece name) {
// We need a null-terminated string to pass to folly::setThreadName().
// The pthread name can be at most 15 bytes long, so truncate it if necessary
// when creating the string.
size_t pthreadLength = std::min(name.size(), (size_t)15);
char pthreadName[pthreadLength + 1];
memcpy(pthreadName, name.begin(), pthreadLength);
pthreadName[pthreadLength] = '\0';
folly::setThreadName(pthreadName);
}
// Helper function to check if URL has valid scheme.
// http_parser only support full form scheme with double slash,
// and the scheme must be all alphabetic charecter.
static bool validateScheme(folly::StringPiece url) {
auto schemeEnd = url.find("://");
if (schemeEnd == std::string::npos || schemeEnd == 0) {
return false;
}
auto scheme = url.subpiece(0, schemeEnd);
return std::all_of(scheme.begin(), scheme.end(), [](auto _) {
return std::isalpha(_);
});
}
void DebugProtocolWriter::writeSP(folly::StringPiece str) {
static constexpr size_t kStringLimit = 256;
static constexpr size_t kStringPrefixSize = 128;
std::string toShow = str.str();
if (toShow.length() > kStringLimit) {
toShow = str.subpiece(0, kStringPrefixSize).str();
folly::toAppend("[...](", str.size(), ")", &toShow);
}
writeItem("\"{}\"", folly::cEscape<std::string>(toShow));
}
/**
* Matches all the occurences of "pattern" in "text"
*
* @return A vector of pairs containing the index and size (respectively)
* of all ocurrences.
*/
std::vector<std::pair<size_t, size_t>> MessagePrinter::matchAll(
folly::StringPiece text, const boost::regex& pattern) const {
std::vector<std::pair<size_t, size_t>> result;
boost::cregex_token_iterator it(text.begin(), text.end(), pattern);
boost::cregex_token_iterator end;
while (it != end) {
result.emplace_back(it->first - text.begin(), it->length());
++it;
}
return result;
}
uint32_t HPACKEncodeBuffer::encodeLiteral(uint8_t instruction, uint8_t nbit,
folly::StringPiece literal) {
if (huffmanEnabled_) {
return encodeHuffman(instruction, nbit, literal);
}
// otherwise use simple layout
uint32_t count =
encodeInteger(literal.size(), instruction, nbit);
// copy the entire string
buf_.push((uint8_t*)literal.data(), literal.size());
count += literal.size();
return count;
}
int8_t getChromeVersion(folly::StringPiece agent) {
static const std::string search = "Chrome/";
auto found = agent.find(search);
int8_t num = -1;
VLOG(5) << "The agent is " << agent << " and found is " << found;
if (found != std::string::npos) {
auto startNum = found + search.length();
if (agent.size() > startNum + 3) {
num = (agent[startNum] - '0') * 10 + (agent[startNum + 1] - '0');
}
}
return num;
}
bool getShardId(folly::StringPiece key, folly::StringPiece& shardId) {
size_t colon = qfind(key, ':');
if (colon == std::string::npos) {
return false;
}
key.advance(colon + 1);
colon = qfind(key, ':');
if (colon == std::string::npos) {
return false;
}
if (colon <= 0 || !isdigit(key[colon - 1])) {
return false;
}
shardId = key.subpiece(0, colon);
return true;
}
PoolFactory::PoolJson PoolFactory::parseNamedPool(folly::StringPiece name) {
auto existingIt = pools_.find(name);
if (existingIt == pools_.end()) {
// get the pool from ConfigApi
std::string jsonStr;
checkLogic(configApi_.get(ConfigType::Pool, name.str(), jsonStr),
"Can not read pool: {}", name);
existingIt = pools_.emplace(
name,
std::make_pair(parseJsonString(jsonStr), PoolState::PARSED)
).first;
return PoolJson(existingIt->first, existingIt->second.first);
}
name = existingIt->first;
auto& json = existingIt->second.first;
auto& state = existingIt->second.second;
switch (state) {
case PoolState::PARSED: return PoolJson(name, json);
case PoolState::PARSING: throwLogic("Cycle in pool inheritance");
case PoolState::NEW:
state = PoolState::PARSING;
break;
}
if (auto jInherit = json.get_ptr("inherit")) {
checkLogic(jInherit->isString(),
"Pool {}: inherit is not a string", name);
auto& newJson = parseNamedPool(jInherit->stringPiece()).json;
json.update_missing(newJson);
json.erase("inherit");
}
state = PoolState::PARSED;
return PoolJson(name, json);
}
std::string FailoverWithExptimeRoute::keyWithFailoverTag(
folly::StringPiece fullKey,
const AccessPoint& ap) {
const size_t tagLength =
kFailoverTagStart.size() +
ap.getHost().size() +
6; // 1 for kFailoverHostPortSeparator + 5 for port.
std::string failoverTag;
failoverTag.reserve(tagLength);
failoverTag = kFailoverTagStart;
failoverTag += ap.getHost().str();
if (ap.getPort() != 0) {
failoverTag += kFailoverHostPortSeparator;
failoverTag += folly::to<std::string>(ap.getPort());
}
// Safety check: scrub the host and port for ':' to avoid appending
// more than one field to the key.
// Note: we start after the ':failover=' part of the string,
// since we need the initial ':' and we know the remainder is safe.
for (size_t i = kFailoverTagStart.size(); i < failoverTag.size(); i++) {
if (failoverTag[i] == ':') {
failoverTag[i] = '$';
}
}
return fullKey.str() + failoverTag;
}
std::vector<std::string> LoggerDB::processConfigString(
folly::StringPiece config) {
std::vector<std::string> errors;
if (config.empty()) {
return errors;
}
std::vector<StringPiece> pieces;
folly::split(",", config, pieces);
for (const auto& p : pieces) {
auto idx = p.rfind('=');
if (idx == folly::StringPiece::npos) {
errors.emplace_back(
folly::sformat("missing '=' in logger configuration: \"{}\"", p));
continue;
}
auto category = p.subpiece(0, idx);
auto level_str = p.subpiece(idx + 1);
LogLevel level;
try {
level = stringToLogLevel(level_str);
} catch (const std::exception&) {
errors.emplace_back(folly::sformat(
"invalid log level \"{}\" for category \"{}\"", level_str, category));
continue;
}
setLevel(category, level);
}
return errors;
}
void utilCreateDir(folly::StringPiece path) {
try {
boost::filesystem::create_directories(path.str());
} catch (...) {
throw SysError(errno, "failed to create directory \"", path, "\"");
}
}
/**
* @return: (item, 0) On a hit.
* (stale_item, token) On a miss.
* (stale_item, 1) On a hot miss (another lease outstanding).
*/
std::pair<MockMc::Item*, uint64_t> MockMc::leaseGet(folly::StringPiece key) {
auto it = findUnexpired(key);
if (it == citems_.end()) {
/* Lease get on a non-existing item: create a new empty item and
put it in TLRU with valid token */
it = citems_.insert(
std::make_pair(key.str(),
CacheItem(Item(folly::IOBuf::copyBuffer(""))))).first;
it->second.state = CacheItem::TLRU;
it->second.updateToken();
}
auto& citem = it->second;
switch (citem.state) {
case CacheItem::CACHE:
/* Regular hit */
return std::make_pair(&citem.item, 0);
case CacheItem::TLRU:
/* First lease-get for a TLRU item, return with a valid token */
citem.state = CacheItem::TLRU_HOT;
return std::make_pair(&citem.item, citem.token);
case CacheItem::TLRU_HOT:
/* TLRU item with other lease-gets pending, return a hot miss token
(special value 1). Note: in real memcached this state would
revert to TLRU after a timeout. */
return std::make_pair(&citem.item, 1);
}
CHECK(false);
}
// Starts tracing using the given profiler
static void start_tracing(XDebugProfiler* profiler,
folly::StringPiece filename = folly::StringPiece(),
int64_t options = 0) {
// Add ini settings
if (XDEBUG_GLOBAL(TraceOptions)) {
options |= k_XDEBUG_TRACE_APPEND;
}
if (XDEBUG_GLOBAL(TraceFormat) == 1) {
options |= k_XDEBUG_TRACE_COMPUTERIZED;
}
if (XDEBUG_GLOBAL(TraceFormat) == 2) {
options |= k_XDEBUG_TRACE_HTML;
}
// If no filename is passed, php5 xdebug stores in the default output
// directory with the default file name.
folly::StringPiece dirname;
if (filename.empty()) {
auto& default_dirname = XDEBUG_GLOBAL(TraceOutputDir);
auto& default_filename = XDEBUG_GLOBAL(TraceOutputName);
dirname = folly::StringPiece(default_dirname);
filename = folly::StringPiece(default_filename);
}
auto const suffix = !(options & k_XDEBUG_TRACE_NAKED_FILENAME);
auto abs_filename = format_filename(dirname, filename, suffix);
profiler->enableTracing(abs_filename, options);
}
std::string toUpper(folly::StringPiece s) {
std::string ret;
ret.reserve(s.size());
for (auto const c : s) {
ret += toupper(c);
}
return ret;
}
std::string Md5Hash(folly::StringPiece input) {
unsigned char result[MD5_DIGEST_LENGTH];
MD5(reinterpret_cast<const unsigned char*>(input.data()), input.size(),
result);
std::string ret;
const std::string kHexBytes = "0123456789abcdef";
for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) {
// Hex should print, for this, in a weird bytewise order:
// If the bytes are b0b1b2b3..., we print:
// hi(b0) lo(b0) hi(b1) lo(b1) ...
// Where hi(x) is the hex char of its upper 4 bits, and lo(x) is lower 4
ret += kHexBytes[(result[i] >> 4) & 0x0F];
ret += kHexBytes[result[i] & 0x0F];
}
return ret;
}
void PktCaptureManager::checkCaptureName(folly::StringPiece name) {
// We use the capture name for the on-disk filename, so don't allow
// directory separator characters or nul bytes.
std::string invalidChars("\0/", 2);
size_t idx = name.find_first_of(invalidChars);
if (idx != std::string::npos) {
throw FbossError("invalid capture name \"", name,
"\": invalid character at byte ", idx);
}
}
void HTTPHeaders::add(folly::StringPiece name, folly::StringPiece value) {
CHECK(name.size());
const HTTPHeaderCode code = HTTPCommonHeaders::hash(name.data(), name.size());
codes_.push_back(code);
headerNames_.push_back((code == HTTP_HEADER_OTHER)
? new std::string(name.data(), name.size())
: HTTPCommonHeaders::getPointerToHeaderName(code));
headerValues_.emplace_back(value.data(), value.size());
}