RPCRequestHandler::RPCRequestHandler() : m_count(0), m_reset(false) {
hphp_session_init();
m_context = hphp_context_init();
m_created = time(0);
Logger::ResetRequestCount();
Logger::Info("creating new RPC request handler");
}
void AdminRequestHandler::setupRequest(Transport* transport) {
auto const cmd = transport->getCommand();
if (strncmp(cmd.c_str(), "dump-apc", 8) == 0) {
hphp_session_init();
} else {
g_context.getCheck();
}
GetAccessLog().onNewRequest();
}
RPCRequestHandler::RPCRequestHandler(bool info /* = true */)
: m_count(0), m_reset(false),
m_returnEncodeType(Json) {
hphp_session_init();
m_context = hphp_context_init();
m_created = time(0);
Logger::ResetRequestCount();
if (info) {
Logger::Info("creating new RPC request handler");
}
}
void preloadRepo() {
auto& repo = Repo::get();
auto units = repo.enumerateUnits(RepoIdLocal, true, false);
if (units.size() == 0) {
units = repo.enumerateUnits(RepoIdCentral, true, false);
}
if (!units.size()) return;
std::vector<std::thread> workers;
auto numWorkers = Process::GetCPUCount();
// Compute a batch size that causes each thread to process approximately 16
// batches. Even if the batches are somewhat imbalanced in what they contain,
// the straggler workers are very unlikey to take more than 10% longer than
// the first worker to finish.
size_t batchSize{std::max(units.size() / numWorkers / 16, size_t(1))};
std::atomic<size_t> index{0};
for (auto worker = 0; worker < numWorkers; ++worker) {
workers.push_back(std::thread([&] {
hphp_session_init();
hphp_context_init();
while (true) {
auto begin = index.fetch_add(batchSize);
auto end = std::min(begin + batchSize, units.size());
if (begin >= end) break;
auto unitCount = end - begin;
for (auto i = size_t{0}; i < unitCount; ++i) {
auto& kv = units[begin + i];
try {
lookupUnit(String(RuntimeOption::SourceRoot + kv.first).get(),
"", nullptr);
} catch (...) {
// swallow errors silently
}
}
}
hphp_context_exit();
hphp_session_exit();
hphp_thread_exit();
}));
}
for (auto& worker : workers) {
worker.join();
}
}
RPCRequestHandler::RPCRequestHandler(bool info /* = true */)
: m_count(0), m_reset(false),
m_returnEncodeType(Json) {
hphp_session_init();
bool isServer = RuntimeOption::serverExecutionMode();
if (isServer) {
m_context = hphp_context_init();
} else {
// In command line mode, we want the xbox workers to
// output to STDOUT
m_context = g_context.getNoCheck();
m_context->obSetImplicitFlush(true);
}
m_created = time(0);
Logger::ResetRequestCount();
if (info) {
Logger::Info("creating new RPC request handler");
}
}
void RPCRequestHandler::initState() {
hphp_session_init();
bool isServer = RuntimeOption::ServerExecutionMode();
if (isServer) {
m_context = hphp_context_init();
} else {
// In command line mode, we want the xbox workers to
// output to STDOUT
m_context = g_context.getNoCheck();
m_context->obSetImplicitFlush(true);
}
m_lastReset = time(0);
Logger::ResetRequestCount();
if (m_logResets) {
Logger::Info("initializing RPC request handler");
}
m_reset = false;
m_requestsSinceReset = 0;
}
void HttpRequestHandler::handleRequest(Transport *transport) {
ExecutionProfiler ep(ThreadInfo::RuntimeFunctions);
Logger::OnNewRequest();
GetAccessLog().onNewRequest();
transport->enableCompression();
ServerStatsHelper ssh("all", ServerStatsHelper::TRACK_MEMORY);
Logger::Verbose("receiving %s", transport->getCommand().c_str());
// will clear all extra logging when this function goes out of scope
StackTraceNoHeap::ExtraLoggingClearer clearer;
StackTraceNoHeap::AddExtraLogging("URL", transport->getUrl());
// resolve virtual host
const VirtualHost *vhost = HttpProtocol::GetVirtualHost(transport);
assert(vhost);
if (vhost->disabled() ||
vhost->isBlocking(transport->getCommand(), transport->getRemoteHost())) {
transport->sendString("Not Found", 404);
return;
}
// don't serve the request if it's been sitting in queue for longer than our
// allowed request timeout.
int requestTimeoutSeconds =
vhost->getRequestTimeoutSeconds(getDefaultTimeout());
if (requestTimeoutSeconds > 0) {
timespec now;
Timer::GetMonotonicTime(now);
const timespec& queueTime = transport->getQueueTime();
if (gettime_diff_us(queueTime, now) > requestTimeoutSeconds * 1000000) {
transport->sendString("Service Unavailable", 503);
m_requestTimedOutOnQueue->addValue(1);
return;
}
}
ServerStats::StartRequest(transport->getCommand().c_str(),
transport->getRemoteHost(),
vhost->getName().c_str());
// resolve source root
string host = transport->getHeader("Host");
SourceRootInfo sourceRootInfo(host.c_str());
if (sourceRootInfo.error()) {
sourceRootInfo.handleError(transport);
return;
}
// request URI
string pathTranslation = m_pathTranslation ?
vhost->getPathTranslation().c_str() : "";
RequestURI reqURI(vhost, transport, sourceRootInfo.path(), pathTranslation);
if (reqURI.done()) {
return; // already handled with redirection or 404
}
string path = reqURI.path().data();
string absPath = reqURI.absolutePath().data();
// determine whether we should compress response
bool compressed = transport->decideCompression();
const char *data; int len;
const char *ext = reqURI.ext();
if (reqURI.forbidden()) {
transport->sendString("Forbidden", 403);
return;
}
bool cachableDynamicContent =
(!RuntimeOption::StaticFileGenerators.empty() &&
RuntimeOption::StaticFileGenerators.find(path) !=
RuntimeOption::StaticFileGenerators.end());
// If this is not a php file, check the static and dynamic content caches
if (ext && strcasecmp(ext, "php") != 0) {
if (RuntimeOption::EnableStaticContentCache) {
bool original = compressed;
// check against static content cache
if (StaticContentCache::TheCache.find(path, data, len, compressed)) {
Util::ScopedMem decompressed_data;
// (qigao) not calling stat at this point because the timestamp of
// local cache file is not valuable, maybe misleading. This way
// the Last-Modified header will not show in response.
// stat(RuntimeOption::FileCache.c_str(), &st);
if (!original && compressed) {
data = gzdecode(data, len);
if (data == nullptr) {
throw FatalErrorException("cannot unzip compressed data");
}
decompressed_data = const_cast<char*>(data);
compressed = false;
}
sendStaticContent(transport, data, len, 0, compressed, path, ext);
StaticContentCache::TheFileCache->adviseOutMemory();
ServerStats::LogPage(path, 200);
GetAccessLog().log(transport, vhost);
return;
}
}
if (RuntimeOption::EnableStaticContentFromDisk) {
String translated = File::TranslatePath(String(absPath));
if (!translated.empty()) {
CstrBuffer sb(translated.data());
if (sb.valid()) {
struct stat st;
st.st_mtime = 0;
stat(translated.data(), &st);
sendStaticContent(transport, sb.data(), sb.size(), st.st_mtime,
false, path, ext);
ServerStats::LogPage(path, 200);
GetAccessLog().log(transport, vhost);
return;
}
}
}
// check static contents that were generated by dynamic pages
if (cachableDynamicContent) {
// check against dynamic content cache
assert(transport->getUrl());
string key = path + transport->getUrl();
if (DynamicContentCache::TheCache.find(key, data, len, compressed)) {
sendStaticContent(transport, data, len, 0, compressed, path, ext);
ServerStats::LogPage(path, 200);
GetAccessLog().log(transport, vhost);
return;
}
}
}
// proxy any URLs that not specified in ServeURLs
if (!RuntimeOption::ProxyOrigin.empty() &&
((RuntimeOption::UseServeURLs &&
RuntimeOption::ServeURLs.find(path) ==
RuntimeOption::ServeURLs.end()) ||
(RuntimeOption::UseProxyURLs &&
(RuntimeOption::ProxyURLs.find(path) !=
RuntimeOption::ProxyURLs.end() ||
MatchAnyPattern(path, RuntimeOption::ProxyPatterns) ||
(abs(rand()) % 100) < RuntimeOption::ProxyPercentage)))) {
for (int i = 0; i < RuntimeOption::ProxyRetry; i++) {
bool force = (i == RuntimeOption::ProxyRetry - 1); // last one
if (handleProxyRequest(transport, force)) break;
}
return;
}
// record request for debugging purpose
std::string tmpfile = HttpProtocol::RecordRequest(transport);
// main body
hphp_session_init();
ThreadInfo::s_threadInfo->m_reqInjectionData.
setTimeout(requestTimeoutSeconds);
bool ret = false;
try {
ret = executePHPRequest(transport, reqURI, sourceRootInfo,
cachableDynamicContent);
} catch (const Eval::DebuggerException &e) {
transport->sendString(e.what(), 200);
transport->onSendEnd();
hphp_context_exit(g_context.getNoCheck(), true, true, transport->getUrl());
} catch (...) {
Logger::Error("Unhandled exception in HPHP server engine.");
}
GetAccessLog().log(transport, vhost);
/*
* HPHP logs may need to access data in ServerStats, so we have to
* clear the hashtable after writing the log entry.
*/
ServerStats::Reset();
hphp_session_exit();
HttpProtocol::ClearRecord(ret, tmpfile);
}
void OptWorker<Pre>::onThreadEnter() {
hphp_session_init();
hphp_context_init();
}
/*
* This is the entry point for offline bytecode generation.
*/
void emitAllHHBC(AnalysisResultPtr&& ar) {
auto ues = ar->getHhasFiles();
decltype(ues) ues_to_print;
auto const outputPath = ar->getOutputPath();
std::thread wp_thread, dispatcherThread;
auto unexpectedException = [&] (const char* what) {
if (dispatcherThread.joinable()) {
Logger::Error("emitAllHHBC exited via an exception "
"before dispatcherThread was joined: %s", what);
}
if (wp_thread.joinable()) {
Logger::Error("emitAllHHBC exited via an exception "
"before wp_thread was joined: %s", what);
}
throw;
};
try {
{
SCOPE_EXIT {
genText(ues_to_print, outputPath);
};
auto commitSome = [&] (decltype(ues)& emitters) {
batchCommit(emitters);
if (Option::GenerateTextHHBC || Option::GenerateHhasHHBC) {
std::move(emitters.begin(), emitters.end(),
std::back_inserter(ues_to_print));
}
emitters.clear();
};
if (!RuntimeOption::EvalUseHHBBC && ues.size()) {
commitSome(ues);
}
auto commitLoop = [&] {
folly::Optional<Timer> commitTime;
// kBatchSize needs to strike a balance between reducing
// transaction commit overhead (bigger batches are better), and
// limiting the cost incurred by failed commits due to identical
// units that require rollback and retry (smaller batches have
// less to lose). Empirical results indicate that a value in
// the 2-10 range is reasonable.
static const unsigned kBatchSize = 8;
while (auto ue = s_ueq.pop()) {
if (!commitTime) {
commitTime.emplace(Timer::WallTime, "committing units to repo");
}
ues.push_back(std::move(ue));
if (ues.size() == kBatchSize) {
commitSome(ues);
}
}
if (ues.size()) commitSome(ues);
};
LitstrTable::get().setReading();
ar->finish();
ar.reset();
if (!RuntimeOption::EvalUseHHBBC) {
if (Option::GenerateBinaryHHBC) {
commitGlobalData(std::unique_ptr<ArrayTypeTable::Builder>{});
}
return;
}
RuntimeOption::EvalJit = false; // For HHBBC to invoke builtins.
std::unique_ptr<ArrayTypeTable::Builder> arrTable;
wp_thread = std::thread([&] {
Timer timer(Timer::WallTime, "running HHBBC");
hphp_thread_init();
hphp_session_init(Treadmill::SessionKind::CompilerEmit);
SCOPE_EXIT {
hphp_context_exit();
hphp_session_exit();
hphp_thread_exit();
};
HHBBC::whole_program(
std::move(ues), s_ueq, arrTable,
Option::ParserThreadCount > 0 ? Option::ParserThreadCount : 0);
});
commitLoop();
commitGlobalData(std::move(arrTable));
}
wp_thread.join();
} catch (std::exception& ex) {
void HttpRequestHandler::handleRequest(Transport *transport) {
Logger::OnNewRequest();
GetAccessLog().onNewRequest();
transport->enableCompression();
Logger::Verbose("receiving %s", transport->getCommand().c_str());
ServerStatsHelper ssh("all", true);
// resolve virtual host
const VirtualHost *vhost = HttpProtocol::GetVirtualHost(transport);
ASSERT(vhost);
if (vhost->disabled() ||
vhost->isBlocking(transport->getCommand(), transport->getRemoteHost())) {
transport->sendString("Not Found", 404);
return;
}
ServerStats::StartRequest(transport->getCommand().c_str(),
transport->getRemoteHost(),
vhost->getName().c_str());
// resolve source root
string host = transport->getHeader("Host");
SourceRootInfo sourceRootInfo(host.c_str());
if (sourceRootInfo.error()) {
sourceRootInfo.handleError(transport);
return;
}
// request URI
string pathTranslation = m_pathTranslation ?
vhost->getPathTranslation().c_str() : "";
RequestURI reqURI(vhost, transport, sourceRootInfo.path(), pathTranslation);
if (reqURI.done()) {
return; // already handled with redirection or 404
}
string path = reqURI.path().data();
string absPath = reqURI.absolutePath().data();
bool compressed = transport->acceptEncoding("gzip");
const char *data; int len;
size_t pos = path.rfind('.');
const char *ext = (pos != string::npos) ? (path.c_str() + pos + 1) : NULL;
bool cachableDynamicContent =
(!RuntimeOption::StaticFileGenerators.empty() &&
RuntimeOption::StaticFileGenerators.find(path) !=
RuntimeOption::StaticFileGenerators.end());
// If this is not a php file, check the static and dynamic content caches
if (ext && strcasecmp(ext, "php") != 0) {
if (RuntimeOption::EnableStaticContentCache) {
// check against static content cache
if (StaticContentCache::TheCache.find(path, data, len, compressed)) {
struct stat st;
st.st_mtime = 0;
// (qigao) not calling stat at this point because the timestamp of
// local cache file is not valuable, maybe misleading. This way
// the Last-Modified header will not show in response.
// stat(RuntimeOption::FileCache.c_str(), &st);
sendStaticContent(transport, data, len, st.st_mtime, compressed, path);
ServerStats::LogPage(path, 200);
return;
}
}
if (RuntimeOption::EnableStaticContentFromDisk &&
RuntimeOption::StaticFileExtensions.find(ext) !=
RuntimeOption::StaticFileExtensions.end()) {
String translated = File::TranslatePath(String(absPath));
if (!translated.empty()) {
StringBuffer sb(translated.data());
if (sb.valid()) {
struct stat st;
st.st_mtime = 0;
stat(translated.data(), &st);
sendStaticContent(transport, sb.data(), sb.size(), st.st_mtime,
false, path);
ServerStats::LogPage(path, 200);
return;
}
}
}
// check static contents that were generated by dynamic pages
if (cachableDynamicContent) {
// check against dynamic content cache
ASSERT(transport->getUrl());
string key = path + transport->getUrl();
if (DynamicContentCache::TheCache.find(key, data, len, compressed)) {
sendStaticContent(transport, data, len, 0, compressed, path);
ServerStats::LogPage(path, 200);
return;
}
}
}
// proxy any URLs that not specified in ServeURLs
if (!RuntimeOption::ProxyOrigin.empty() &&
((RuntimeOption::UseServeURLs &&
RuntimeOption::ServeURLs.find(path) ==
RuntimeOption::ServeURLs.end()) ||
(RuntimeOption::UseProxyURLs &&
(RuntimeOption::ProxyURLs.find(path) !=
RuntimeOption::ProxyURLs.end() ||
MatchAnyPattern(path, RuntimeOption::ProxyPatterns) ||
(abs(rand()) % 100) < RuntimeOption::ProxyPercentage)))) {
for (int i = 0; i < RuntimeOption::ProxyRetry; i++) {
bool force = (i == RuntimeOption::ProxyRetry - 1); // last one
if (handleProxyRequest(transport, force)) break;
}
return;
}
// record request for debugging purpose
std::string tmpfile = HttpProtocol::RecordRequest(transport);
// main body
hphp_session_init();
bool ret = false;
try {
ret = executePHPRequest(transport, reqURI, sourceRootInfo,
cachableDynamicContent);
} catch (...) {
Logger::Error("Unhandled exception in HPHP server engine.");
}
GetAccessLog().log(transport);
hphp_session_exit();
HttpProtocol::ClearRecord(ret, tmpfile);
}
void HttpRequestHandler::handleRequest(Transport *transport) {
ExecutionProfiler ep(ThreadInfo::RuntimeFunctions);
Logger::OnNewRequest();
GetAccessLog().onNewRequest();
transport->enableCompression();
ServerStatsHelper ssh("all", true);
Logger::Verbose("receiving %s", transport->getCommand().c_str());
// will clear all extra logging when this function goes out of scope
StackTraceNoHeap::ExtraLoggingClearer clearer;
StackTraceNoHeap::AddExtraLogging("URL", transport->getUrl());
// resolve virtual host
const VirtualHost *vhost = HttpProtocol::GetVirtualHost(transport);
ASSERT(vhost);
if (vhost->disabled() ||
vhost->isBlocking(transport->getCommand(), transport->getRemoteHost())) {
transport->sendString("Not Found", 404);
return;
}
ServerStats::StartRequest(transport->getCommand().c_str(),
transport->getRemoteHost(),
vhost->getName().c_str());
// resolve source root
string host = transport->getHeader("Host");
SourceRootInfo sourceRootInfo(host.c_str());
if (sourceRootInfo.error()) {
sourceRootInfo.handleError(transport);
return;
}
// request URI
string pathTranslation = m_pathTranslation ?
vhost->getPathTranslation().c_str() : "";
RequestURI reqURI(vhost, transport, sourceRootInfo.path(), pathTranslation);
if (reqURI.done()) {
return; // already handled with redirection or 404
}
string path = reqURI.path().data();
string absPath = reqURI.absolutePath().data();
// determine whether we should compress response
bool compressed = transport->decideCompression();
const char *data; int len;
size_t pos = path.rfind('.');
const char *ext =
(pos != string::npos) &&
path.find('/', pos) == string::npos // no extention in ./foo or ../bar
? (path.c_str() + pos + 1) : NULL;
bool cachableDynamicContent =
(!RuntimeOption::StaticFileGenerators.empty() &&
RuntimeOption::StaticFileGenerators.find(path) !=
RuntimeOption::StaticFileGenerators.end());
// If this is not a php file, check the static and dynamic content caches
if (ext && strcasecmp(ext, "php") != 0) {
if (RuntimeOption::EnableStaticContentCache) {
bool original = compressed;
// check against static content cache
if (StaticContentCache::TheCache.find(path, data, len, compressed)) {
struct stat st;
st.st_mtime = 0;
String str;
// (qigao) not calling stat at this point because the timestamp of
// local cache file is not valuable, maybe misleading. This way
// the Last-Modified header will not show in response.
// stat(RuntimeOption::FileCache.c_str(), &st);
if (!original && compressed) {
data = gzdecode(data, len);
if (data == NULL) {
throw FatalErrorException("cannot unzip compressed data");
}
compressed = false;
str = NEW(StringData)(data, len, AttachString);
}
sendStaticContent(transport, data, len, st.st_mtime, compressed, path);
StaticContentCache::TheFileCache->adviseOutMemory();
ServerStats::LogPage(path, 200);
return;
}
}
if (RuntimeOption::EnableStaticContentFromDisk) {
String translated = File::TranslatePath(String(absPath));
if (!translated.empty()) {
StringBuffer sb(translated.data());
if (sb.valid()) {
struct stat st;
st.st_mtime = 0;
stat(translated.data(), &st);
sendStaticContent(transport, sb.data(), sb.size(), st.st_mtime,
false, path);
ServerStats::LogPage(path, 200);
return;
}
}
}
// check static contents that were generated by dynamic pages
if (cachableDynamicContent) {
// check against dynamic content cache
ASSERT(transport->getUrl());
string key = path + transport->getUrl();
if (DynamicContentCache::TheCache.find(key, data, len, compressed)) {
sendStaticContent(transport, data, len, 0, compressed, path);
ServerStats::LogPage(path, 200);
return;
}
}
}
// proxy any URLs that not specified in ServeURLs
if (!RuntimeOption::ProxyOrigin.empty() &&
((RuntimeOption::UseServeURLs &&
RuntimeOption::ServeURLs.find(path) ==
RuntimeOption::ServeURLs.end()) ||
(RuntimeOption::UseProxyURLs &&
(RuntimeOption::ProxyURLs.find(path) !=
RuntimeOption::ProxyURLs.end() ||
MatchAnyPattern(path, RuntimeOption::ProxyPatterns) ||
(abs(rand()) % 100) < RuntimeOption::ProxyPercentage)))) {
for (int i = 0; i < RuntimeOption::ProxyRetry; i++) {
bool force = (i == RuntimeOption::ProxyRetry - 1); // last one
if (handleProxyRequest(transport, force)) break;
}
return;
}
// record request for debugging purpose
std::string tmpfile = HttpProtocol::RecordRequest(transport);
// main body
hphp_session_init();
vhost->setRequestTimeoutSeconds();
bool ret = false;
try {
ret = executePHPRequest(transport, reqURI, sourceRootInfo,
cachableDynamicContent);
} catch (const Eval::DebuggerException &e) {
transport->sendString(e.what(), 200);
transport->onSendEnd();
hphp_context_exit(g_context.getNoCheck(), true, true, transport->getUrl());
} catch (...) {
Logger::Error("Unhandled exception in HPHP server engine.");
}
GetAccessLog().log(transport, vhost);
hphp_session_exit();
HttpProtocol::ClearRecord(ret, tmpfile);
}
// (PHP-CALLBACK entry-point) This manually gets a lock where needed but
// avoids holding one most of the time as this can be a quite slow operation.
void runCallback() {
hphp_session_init(Treadmill::SessionKind::Watchman);
auto context = g_context.getNoCheck();
SCOPE_EXIT {
hphp_context_exit();
hphp_session_exit();
{
std::lock_guard<std::mutex> g(s_sharedDataMutex);
processNextUpdate();
}
};
try {
std::string json_data;
{
std::lock_guard<std::mutex> g(s_sharedDataMutex);
if (m_unprocessedCallbackData.empty()) {
return;
}
auto& data = m_unprocessedCallbackData.back();
json_data = toJson(data);
m_unprocessedCallbackData.pop_back();
}
bool initial;
auto unit = lookupUnit(
String(m_callbackFile.c_str()).get(),
"",
&initial,
Native::s_noNativeFuncs);
if (!unit) {
throw std::runtime_error(
folly::sformat("Unit '{}' no longer exists.", m_callbackFile));
}
auto unit_result = Variant::attach(context->invokeUnit(unit));
auto func = Unit::loadFunc(String(m_callbackFunc.c_str()).get());
if (!func) {
throw std::runtime_error(
folly::sformat("Callback '{}' no longer exists", m_callbackFunc));
}
String str_path(m_path.c_str());
String str_query(m_query.c_str());
String str_name(m_name.c_str());
String str_json_data(json_data.c_str());
String str_socket_path(m_socketPath.c_str());
TypedValue args[] = {
str_path.toCell(),
str_query.toCell(),
str_name.toCell(),
str_json_data.toCell(),
str_socket_path.toCell(),
};
tvDecRefGen(
context->invokeFuncFew(func,
nullptr, // thisOrCls
nullptr, // invName
5, // argc
args)
);
} catch(Exception& e) {
if (m_error.empty()) {
m_error = e.getMessage();
}
} catch(Object& e) {
if (m_error.empty()) {
try {
m_error = e->invokeToString().data();
} catch(...) {
m_error = "PHP exception which cannot be turned into a string";
}
}
} catch(const std::exception& e) {
if (m_error.empty()) {
m_error = folly::exceptionStr(e).toStdString();
}
} catch(...) {
if (m_error.empty()) {
m_error = "Unknown error (non std::exception)";
}
}
}
RPCRequestHandler::RPCRequestHandler() : m_count(0), m_reset(false) {
hphp_session_init();
m_created = time(0);
}