void helper<ProtocolReader, ProtocolWriter>::process_exn(
const char* func,
const string& msg,
unique_ptr<ResponseChannel::Request> req,
Cpp2RequestContext* ctx,
EventBase* eb,
int32_t protoSeqId) {
ProtocolWriter oprot;
if (req) {
LOG(ERROR) << msg << " in function " << func;
TApplicationException x(msg);
IOBufQueue queue = helper_w<ProtocolWriter>::write_exn(
func, &oprot, protoSeqId, nullptr, x);
queue.append(THeader::transform(
queue.move(),
ctx->getHeader()->getWriteTransforms(),
ctx->getHeader()->getMinCompressBytes()));
auto queue_mw = makeMoveWrapper(move(queue));
auto req_mw = makeMoveWrapper(move(req));
eb->runInEventBaseThread([=]() mutable {
(*req_mw)->sendReply(queue_mw->move());
});
} else {
LOG(ERROR) << msg << " in oneway function " << func;
}
}
std::unique_ptr<IOBuf> LengthFieldBasedFrameDecoder::decode(
Context* ctx, IOBufQueue& buf, size_t&) {
// discarding too long frame
if (buf.chainLength() <= lengthFieldEndOffset_) {
return nullptr;
}
uint64_t frameLength = getUnadjustedFrameLength(
buf, lengthFieldOffset_, lengthFieldLength_, networkByteOrder_);
frameLength += lengthAdjustment_ + lengthFieldEndOffset_;
if (frameLength < lengthFieldEndOffset_) {
throw std::runtime_error("Frame too small");
}
if (frameLength > maxFrameLength_) {
throw std::runtime_error("Frame larger than " +
folly::to<std::string>(maxFrameLength_));
}
if (buf.chainLength() < frameLength) {
return nullptr;
}
if (initialBytesToStrip_ > frameLength) {
throw std::runtime_error("InitialBytesToSkip larger than frame");
}
buf.trimStart(initialBytesToStrip_);
int actualFrameLength = frameLength - initialBytesToStrip_;
return buf.split(actualFrameLength);
}
bool decode(Context*,
IOBufQueue& buf,
std::string& result,
size_t&) override {
if (buf.chainLength() > 0) {
result = buf.move()->moveToFbString().toStdString();
return true;
}
return false;
}
void FastCGITransport::sendResponseHeaders(IOBufQueue& queue, int code) {
CHECK(!m_headersSent);
m_headersSent = true;
if (code != 200) {
queue.append(s_status);
queue.append(std::to_string(code));
auto reasonStr = getResponseInfo();
if (reasonStr.empty()) {
reasonStr = HttpProtocol::GetReasonString(code);
}
queue.append(s_space);
queue.append(reasonStr);
queue.append(s_newline);
}
for (auto& header : m_responseHeaders) {
for (auto& value : header.second) {
queue.append(header.first);
queue.append(s_colon);
queue.append(value);
queue.append(s_newline);
}
}
queue.append(s_newline);
}
TEST(SerializationTest, DeserializeReturningObjGivenStringPiece) {
using serializer = SimpleJSONSerializer;
auto s = makeTestStruct();
IOBufQueue q;
serializer::serialize(s, &q);
auto b = q.move();
auto out = serializer::deserialize<TestStruct>(StringPiece(b->coalesce()));
EXPECT_EQ(s, out);
}
TEST_F(BroadcastHandlerTest, OnCompleted) {
// Test with EOF on the handler
EXPECT_CALL(*decoder, decode(_, _, _, _))
.WillRepeatedly(
Invoke([&](MockByteToMessageDecoder<std::string>::Context*,
IOBufQueue& q, std::string& data, size_t&) {
auto buf = q.move();
if (buf) {
buf->coalesce();
data = buf->moveToFbString().toStdString();
return true;
}
return false;
}));
InSequence dummy;
// Add a subscriber
EXPECT_EQ(handler->subscribe(&subscriber0), 0);
EXPECT_CALL(subscriber0, onNext("data1")).Times(1);
// Push some data
IOBufQueue q;
q.append(IOBuf::copyBuffer("data1"));
pipeline->read(q);
q.clear();
// Add another subscriber
EXPECT_EQ(handler->subscribe(&subscriber1), 1);
EXPECT_CALL(subscriber0, onNext("data2")).Times(1);
EXPECT_CALL(subscriber1, onNext("data2")).Times(1);
// Push more data
q.append(IOBuf::copyBuffer("data2"));
pipeline->read(q);
q.clear();
// Unsubscribe one of the subscribers
handler->unsubscribe(0);
EXPECT_CALL(subscriber1, onCompleted()).Times(1);
EXPECT_CALL(*handler, close(_))
.WillOnce(InvokeWithoutArgs([this] {
pipeline.reset();
return makeFuture();
}));
// The handler should be deleted now
handler->readEOF(nullptr);
}
int64_t LineBasedFrameDecoder::findEndOfLine(IOBufQueue& buf) {
Cursor c(buf.front());
for (uint32_t i = 0; i < maxLength_ && i < buf.chainLength(); i++) {
auto b = c.read<char>();
if (b == '\n' && terminatorType_ != TerminatorType::CARRIAGENEWLINE) {
return i;
} else if (terminatorType_ != TerminatorType::NEWLINE &&
b == '\r' && !c.isAtEnd() && c.read<char>() == '\n') {
return i;
}
}
return -1;
}
void readBench(size_t iters) {
BenchmarkSuspender susp;
auto strct = create<Struct>();
IOBufQueue q;
Serializer::serialize(strct, &q);
auto buf = q.move();
susp.dismiss();
while (iters--) {
Struct data;
Serializer::deserialize(buf.get(), data);
}
susp.rehire();
}
void FastCGITransport::sendResponseHeaders(IOBufQueue& queue) {
CHECK(!m_headersSent);
m_headersSent = true;
for (auto header : m_responseHeaders) {
for (auto value : header.second) {
queue.append(header.first);
queue.append(": ");
queue.append(value);
queue.append("\r\n");
}
}
queue.append("\r\n");
}
int readBenchMemory(size_t iters) {
auto strct = create<Struct>();
IOBufQueue q;
Serializer::serialize(strct, &q);
auto buf = q.move();
auto before = getMemoryRequestsCounter();
while (iters--) {
Struct data;
Serializer::deserialize(buf.get(), data);
}
auto after = getMemoryRequestsCounter();
return after - before;
}
void FastCGITransport::sendImpl(const void *data, int size, int code,
bool chunked) {
IOBufQueue queue;
if (!m_headersSent) {
sendResponseHeaders(queue, code);
}
queue.append(IOBuf::copyBuffer(data, size));
folly::MoveWrapper<std::unique_ptr<IOBuf>> chain_wrapper(queue.move());
Callback* callback = m_callback;
auto fn = [callback, chain_wrapper]() mutable {
if (callback) {
callback->onStdOut(std::move(*chain_wrapper));
}
};
m_connection->getEventBase()->runInEventBaseThread(fn);
}
void read(Context* ctx, IOBufQueue& q) override {
auto buf = q.pop_front();
buf->coalesce();
std::string data((const char*)buf->data(), buf->length());
ctx->fireRead(data);
}
bool decode(Context* ctx,
IOBufQueue& buf,
std::unique_ptr<IOBuf>& result,
size_t&) override {
result = buf.move();
return result != nullptr;
}
void FastCGITransport::sendResponseHeaders(IOBufQueue& queue, int code) {
auto appender = [&](folly::StringPiece sp) mutable { queue.append(sp); };
if (code != 200) {
auto info = getResponseInfo();
auto reason = !info.empty() ? info : HttpProtocol::GetReasonString(code);
folly::format("Status: {} {}\r\n", code, reason)(appender);
}
for (auto& header : m_responseHeaders) {
for (auto& value : header.second) {
folly::format("{}: {}\r\n", header.first, value)(appender);
}
}
queue.append("\r\n", 2);
}
std::pair<std::string, std::string> Subprocess::communicate(
StringPiece input) {
IOBufQueue inputQueue;
inputQueue.wrapBuffer(input.data(), input.size());
auto outQueues = communicateIOBuf(std::move(inputQueue));
auto outBufs = std::make_pair(outQueues.first.move(),
outQueues.second.move());
std::pair<std::string, std::string> out;
if (outBufs.first) {
outBufs.first->coalesce();
out.first.assign(reinterpret_cast<const char*>(outBufs.first->data()),
outBufs.first->length());
}
if (outBufs.second) {
outBufs.second->coalesce();
out.second.assign(reinterpret_cast<const char*>(outBufs.second->data()),
outBufs.second->length());
}
return out;
}
bool KVParser::parseKeyValueContent(Cursor& cursor,
size_t& available,
size_t& length,
IOBufQueue& queue) {
std::unique_ptr<IOBuf> buf;
size_t len = cursor.cloneAtMost(buf, length);
queue.append(std::move(buf));
assert(length >= len);
length -= len;
assert(available >= len);
available -= len;
return (length == 0);
}
void
IOBufQueue::append(IOBufQueue& other, bool pack) {
if (!other.head_) {
return;
}
// We're going to chain other, thus we need to grab both guards.
auto otherGuard = other.updateGuard();
auto guard = updateGuard();
if (options_.cacheChainLength) {
if (other.options_.cacheChainLength) {
chainLength_ += other.chainLength_;
} else {
chainLength_ += other.head_->computeChainDataLength();
}
}
appendToChain(head_, std::move(other.head_), pack);
other.chainLength_ = 0;
}
uint64_t LengthFieldBasedFrameDecoder::getUnadjustedFrameLength(
IOBufQueue& buf, int offset, int length, bool networkByteOrder) {
folly::io::Cursor c(buf.front());
uint64_t frameLength;
c.skip(offset);
switch(length) {
case 1:{
if (networkByteOrder) {
frameLength = c.readBE<uint8_t>();
} else {
frameLength = c.readLE<uint8_t>();
}
break;
}
case 2:{
if (networkByteOrder) {
frameLength = c.readBE<uint16_t>();
} else {
frameLength = c.readLE<uint16_t>();
}
break;
}
case 4:{
if (networkByteOrder) {
frameLength = c.readBE<uint32_t>();
} else {
frameLength = c.readLE<uint32_t>();
}
break;
}
case 8:{
if (networkByteOrder) {
frameLength = c.readBE<uint64_t>();
} else {
frameLength = c.readLE<uint64_t>();
}
break;
}
}
return frameLength;
}
void FastCGITransport::sendResponseHeaders(IOBufQueue& queue, int code) {
CHECK(!m_headersSent);
m_headersSent = true;
if (code != 200) {
queue.append(s_status);
queue.append(std::to_string(code));
queue.append(s_newline);
}
for (auto& header : m_responseHeaders) {
for (auto& value : header.second) {
queue.append(header.first);
queue.append(s_colon);
queue.append(value);
queue.append(s_newline);
}
}
queue.append(s_newline);
}
std::pair<IOBufQueue, IOBufQueue> Subprocess::communicateIOBuf(
IOBufQueue input) {
// If the user supplied a non-empty input buffer, make sure
// that stdin is a pipe so we can write the data.
if (!input.empty()) {
// findByChildFd() will throw std::invalid_argument if no pipe for
// STDIN_FILENO exists
findByChildFd(STDIN_FILENO);
}
std::pair<IOBufQueue, IOBufQueue> out;
auto readCallback = [&] (int pfd, int cfd) -> bool {
if (cfd == STDOUT_FILENO) {
return handleRead(pfd, out.first);
} else if (cfd == STDERR_FILENO) {
return handleRead(pfd, out.second);
} else {
// Don't close the file descriptor, the child might not like SIGPIPE,
// just read and throw the data away.
return discardRead(pfd);
}
};
auto writeCallback = [&] (int pfd, int cfd) -> bool {
if (cfd == STDIN_FILENO) {
return handleWrite(pfd, input);
} else {
// If we don't want to write to this fd, just close it.
return true;
}
};
communicate(std::move(readCallback), std::move(writeCallback));
return out;
}
TEST(CommunicateSubprocessTest, Duplex2) {
checkFdLeak([] {
// Pipe 200,000 lines through sed
const size_t numCopies = 100000;
auto iobuf = IOBuf::copyBuffer("this is a test\nanother line\n");
IOBufQueue input;
for (size_t n = 0; n < numCopies; ++n) {
input.append(iobuf->clone());
}
std::vector<std::string> cmd({
"sed", "-u",
"-e", "s/a test/a successful test/",
"-e", "/^another line/w/dev/stderr",
});
auto options = Subprocess::pipeStdin().pipeStdout().pipeStderr().usePath();
Subprocess proc(cmd, options);
auto out = proc.communicateIOBuf(std::move(input));
proc.waitChecked();
// Convert stdout and stderr to strings so we can call split() on them.
fbstring stdoutStr;
if (out.first.front()) {
stdoutStr = out.first.move()->moveToFbString();
}
fbstring stderrStr;
if (out.second.front()) {
stderrStr = out.second.move()->moveToFbString();
}
// stdout should be a copy of stdin, with "a test" replaced by
// "a successful test"
std::vector<StringPiece> stdoutLines;
split('\n', stdoutStr, stdoutLines);
EXPECT_EQ(numCopies * 2 + 1, stdoutLines.size());
// Strip off the trailing empty line
if (!stdoutLines.empty()) {
EXPECT_EQ("", stdoutLines.back());
stdoutLines.pop_back();
}
size_t linenum = 0;
for (const auto& line : stdoutLines) {
if ((linenum & 1) == 0) {
EXPECT_EQ("this is a successful test", line);
} else {
EXPECT_EQ("another line", line);
}
++linenum;
}
// stderr should only contain the lines containing "another line"
std::vector<StringPiece> stderrLines;
split('\n', stderrStr, stderrLines);
EXPECT_EQ(numCopies + 1, stderrLines.size());
// Strip off the trailing empty line
if (!stderrLines.empty()) {
EXPECT_EQ("", stderrLines.back());
stderrLines.pop_back();
}
for (const auto& line : stderrLines) {
EXPECT_EQ("another line", line);
}
});
}
bool LineBasedFrameDecoder::decode(Context* ctx,
IOBufQueue& buf,
std::unique_ptr<IOBuf>& result,
size_t&) {
int64_t eol = findEndOfLine(buf);
if (!discarding_) {
if (eol >= 0) {
Cursor c(buf.front());
c += eol;
auto delimLength = c.read<char>() == '\r' ? 2 : 1;
if (eol > maxLength_) {
buf.split(eol + delimLength);
fail(ctx, folly::to<std::string>(eol));
return false;
}
std::unique_ptr<folly::IOBuf> frame;
if (stripDelimiter_) {
frame = buf.split(eol);
buf.trimStart(delimLength);
} else {
frame = buf.split(eol + delimLength);
}
result = std::move(frame);
return true;
} else {
auto len = buf.chainLength();
if (len > maxLength_) {
discardedBytes_ = len;
buf.trimStart(len);
discarding_ = true;
fail(ctx, "over " + folly::to<std::string>(len));
}
return false;
}
} else {
if (eol >= 0) {
Cursor c(buf.front());
c += eol;
auto delimLength = c.read<char>() == '\r' ? 2 : 1;
buf.trimStart(eol + delimLength);
discardedBytes_ = 0;
discarding_ = false;
} else {
discardedBytes_ = buf.chainLength();
buf.move();
}
return false;
}
}
TEST_F(BroadcastHandlerTest, BufferedRead) {
// Test with decoder that buffers data based on some local logic
// before pushing to subscribers
IOBufQueue bufQueue{IOBufQueue::cacheChainLength()};
EXPECT_CALL(*decoder, decode(_, _, _, _))
.WillRepeatedly(
Invoke([&](MockByteToMessageDecoder<std::string>::Context*,
IOBufQueue& q, std::string& data, size_t&) {
bufQueue.append(q);
if (bufQueue.chainLength() < 5) {
return false;
}
auto buf = bufQueue.move();
buf->coalesce();
data = buf->moveToFbString().toStdString();
return true;
}));
InSequence dummy;
// Add a subscriber
EXPECT_EQ(handler->subscribe(&subscriber0), 0);
EXPECT_CALL(subscriber0, onNext("data1")).Times(1);
// Push some fragmented data
IOBufQueue q;
q.append(IOBuf::copyBuffer("da"));
pipeline->read(q);
q.clear();
q.append(IOBuf::copyBuffer("ta1"));
pipeline->read(q);
q.clear();
// Push more fragmented data. onNext shouldn't be called yet.
q.append(IOBuf::copyBuffer("dat"));
pipeline->read(q);
q.clear();
q.append(IOBuf::copyBuffer("a"));
pipeline->read(q);
q.clear();
// Add another subscriber
EXPECT_EQ(handler->subscribe(&subscriber1), 1);
EXPECT_CALL(subscriber0, onNext("data3data4")).Times(1);
EXPECT_CALL(subscriber1, onNext("data3data4")).Times(1);
// Push rest of the fragmented data. The entire data should be pushed
// to both subscribers.
q.append(IOBuf::copyBuffer("3data4"));
pipeline->read(q);
q.clear();
EXPECT_CALL(subscriber0, onNext("data2")).Times(1);
EXPECT_CALL(subscriber1, onNext("data2")).Times(1);
// Push some unfragmented data
q.append(IOBuf::copyBuffer("data2"));
pipeline->read(q);
q.clear();
EXPECT_CALL(*handler, close(_))
.WillOnce(InvokeWithoutArgs([this] {
pipeline.reset();
return makeFuture();
}));
// Unsubscribe all subscribers. The handler should be deleted now.
handler->unsubscribe(0);
handler->unsubscribe(1);
}
void read(Context* ctx, IOBufQueue& q) override {
frontendPipeline_->write(q.move());
}
void read(Context*, IOBufQueue& q) override {
backendPipeline_->write(q.move());
}