TSimpleFileTransport:: TSimpleFileTransport(const std::string& path, bool read, bool write) : TFDTransport(-1, TFDTransport::CLOSE_ON_DESTROY) { int flags = 0; if (read && write) { flags = O_RDWR; } else if (read) { flags = O_RDONLY; } else if (write) { flags = O_WRONLY; } else { throw TTransportException("Neither READ nor WRITE specified"); } if (write) { flags |= O_CREAT | O_APPEND; } int fd = ::open(path.c_str(), flags, S_IRUSR | S_IWUSR| S_IRGRP | S_IROTH); if (fd < 0) { throw TTransportException("failed to open file for writing: " + path); } setFD(fd); open(); }
/* * request for a page allocation */ FixedSizeMemoryPage* FixedSizeMemoryPageFactory::getPage(bool throwOnError) { FixedSizeMemoryPage* page = nullptr; // lock { SpinLockHolder guard(&lock_); if ((page = cachedPages_) != nullptr) { // cache is available cachedPages_ = cachedPages_->next_; --numCachedPages_; // get from cache } else { // allocate new page // check capacity if (numAllocatedPages_ * pageSize_ >= maxMemoryUsage_) { GlobalOutput.printf("FixedSizeMemoryPage::getPage: alloc %d, max %d", numAllocatedPages_ * pageSize_, maxMemoryUsage_); if (throwOnError) { throw TTransportException(TTransportException::INTERNAL_ERROR); } return nullptr; } page = (FixedSizeMemoryPage*)malloc(pageSize_ // memory itself + sizeof(FixedSizeMemoryPage)); // + object size if (!page) { // no memory available if (throwOnError) { throw TTransportException(TTransportException::INTERNAL_ERROR); } return nullptr; } ++numAllocatedPages_; } } // init page page->next_ = nullptr; return page; }
void TZlibTransport::verifyChecksum() { if (!standalone_) { throw TTransportException( TTransportException::BAD_ARGS, "TZLibTransport can only verify checksums for standalone objects."); } if (!input_ended_) { // This should only be called when reading is complete, // but it's possible that the whole checksum has not been fed to zlib yet. // We try to read an extra byte here to force zlib to finish the stream. // It might not always be easy to "unread" this byte, // but we throw an exception if we get it, which is not really // a recoverable error, so it doesn't matter. uint8_t buf[1]; uint32_t got = this->read(buf, sizeof(buf)); if (got || !input_ended_) { throw TTransportException( TTransportException::CORRUPTED_DATA, "Zlib stream not complete."); } } // If the checksum had been bad, we would have gotten an error while // inflating. }
void TMemoryBuffer::ensureCanWrite(uint32_t len) { // Check available space uint32_t avail = available_write(); if (len <= avail) { return; } if (!owner_) { throw TTransportException("Insufficient space in external MemoryBuffer"); } // Grow the buffer as necessary. while (len > avail) { bufferSize_ *= 2; wBound_ = buffer_ + bufferSize_; avail = available_write(); } // Allocate into a new pointer so we don't bork ours if it fails. void* new_buffer = std::realloc(buffer_, bufferSize_); if (new_buffer == NULL) { throw TTransportException("Out of memory."); } ptrdiff_t offset = (uint8_t*)new_buffer - buffer_; buffer_ += offset; rBase_ += offset; rBound_ += offset; wBase_ += offset; wBound_ += offset; }
TSimpleFileTransport::TSimpleFileTransport(const std::string& path, bool read, bool write, bool append) : TFDTransport(-1, TFDTransport::CLOSE_ON_DESTROY) { int flags = 0; if (read && write) { flags = O_RDWR; } else if (read) { flags = O_RDONLY; } else if (write) { flags = O_WRONLY; } else { throw TTransportException("Neither READ nor WRITE specified"); } flags |= O_BINARY; // unix에는 없지만 windows에서는 넣어주어야 모든 곳에서 사용가능 (O_BINARY아닐 때 \r\n을 붙이는 문제가 발생함.) - joygram(2016/02/01) if (write) { flags |= O_CREAT; if (append) { flags |= O_APPEND; } else if (false == read) { flags |= O_TRUNC; // 읽는 것도 아니고 덧대는 것이 아닌 경우 기존 내용 제거 joygram(2016/02/01) } } #ifndef _WIN32 mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; #else int mode = _S_IREAD | _S_IWRITE; #endif int fd = ::THRIFT_OPEN(path.c_str(), flags, mode); if (fd < 0) { throw TTransportException("failed to open file for writing: " + path); } setFD(fd); open(); }
void TPipe::open() { if (isOpen()) return; TAutoHandle hPipe; do { DWORD flags = FILE_FLAG_OVERLAPPED; // async mode, so we can do reads at the same time as writes hPipe.reset(CreateFile(pipename_.c_str(), GENERIC_READ | GENERIC_WRITE, 0, // no sharing NULL, // default security attributes OPEN_EXISTING, // opens existing pipe flags, NULL)); // no template file if (hPipe.h != INVALID_HANDLE_VALUE) break; // success! if (::GetLastError() != ERROR_PIPE_BUSY) { GlobalOutput.perror("TPipe::open ::CreateFile errored GLE=", ::GetLastError()); throw TTransportException(TTransportException::NOT_OPEN, "Unable to open pipe"); } } while (::WaitNamedPipe(pipename_.c_str(), TimeoutSeconds_ * 1000)); if (hPipe.h == INVALID_HANDLE_VALUE) { GlobalOutput.perror("TPipe::open ::CreateFile errored GLE=", ::GetLastError()); throw TTransportException(TTransportException::NOT_OPEN, "Unable to open pipe"); } impl_.reset(new TNamedPipeImpl(hPipe.h)); hPipe.release(); }
uint32_t TQIODeviceTransport::read(uint8_t* buf, uint32_t len) { uint32_t actualSize; qint64 readSize; if (!dev_->isOpen()) { throw TTransportException(TTransportException::NOT_OPEN, "read(): underlying QIODevice is not open"); } actualSize = (uint32_t)std::min((qint64)len, dev_->bytesAvailable()); readSize = dev_->read(reinterpret_cast<char *>(buf), actualSize); if (readSize < 0) { QAbstractSocket* socket; if ((socket = qobject_cast<QAbstractSocket* >(dev_.get()))) { throw TTransportException(TTransportException::UNKNOWN, "Failed to read() from QAbstractSocket", socket->error()); } throw TTransportException(TTransportException::UNKNOWN, "Failed to read from from QIODevice"); } return (uint32_t)readSize; }
void TFileTransport::seekToChunk(int32_t chunk) { if (fd_ <= 0) { throw TTransportException("File not open"); } int32_t numChunks = getNumChunks(); // file is empty, seeking to chunk is pointless if (numChunks == 0) { return; } // negative indicates reverse seek (from the end) if (chunk < 0) { chunk += numChunks; } // too large a value for reverse seek, just seek to beginning if (chunk < 0) { T_DEBUG("%s", "Incorrect value for reverse seek. Seeking to beginning..."); chunk = 0; } // cannot seek past EOF bool seekToEnd = false; off_t minEndOffset = 0; if (chunk >= numChunks) { T_DEBUG("%s", "Trying to seek past EOF. Seeking to EOF instead..."); seekToEnd = true; chunk = numChunks - 1; // this is the min offset to process events till minEndOffset = lseek(fd_, 0, SEEK_END); } off_t newOffset = off_t(chunk) * chunkSize_; offset_ = lseek(fd_, newOffset, SEEK_SET); readState_.resetAllValues(); currentEvent_ = NULL; if (offset_ == -1) { GlobalOutput("TFileTransport: lseek error in seekToChunk"); throw TTransportException("TFileTransport: lseek error in seekToChunk"); } // seek to EOF if user wanted to go to last chunk if (seekToEnd) { uint32_t oldReadTimeout = getReadTimeout(); setReadTimeout(NO_TAIL_READ_TIMEOUT); // keep on reading unti the last event at point of seekChunk call boost::scoped_ptr<eventInfo> event; while ((offset_ + readState_.bufferPtr_) < minEndOffset) { event.reset(readEvent()); if (event.get() == NULL) { break; } } setReadTimeout(oldReadTimeout); } }
void TZlibTransport::verifyChecksum() { // If zlib has already reported the end of the stream, // it has verified the checksum. if (input_ended_) { return; } // This should only be called when reading is complete. // If the caller still has unread data, throw an exception. if (readAvail() > 0) { throw TTransportException( TTransportException::CORRUPTED_DATA, "verifyChecksum() called before end of zlib stream"); } // Reset the rstream fields, in case avail_out is 0. // (Since readAvail() is 0, we know there is no unread data in urbuf_) rstream_->next_out = urbuf_; rstream_->avail_out = urbuf_size_; urpos_ = 0; // Call inflate() // This will throw an exception if the checksum is bad. bool performed_inflate = readFromZlib(); if (!performed_inflate) { // We needed to read from the underlying transport, and the read() call // returned 0. // // Not all TTransport implementations behave the same way here, so we'll // end up with different behavior depending on the underlying transport. // // For some transports (e.g., TFDTransport), read() blocks if no more data // is available. They only return 0 if EOF has been reached, or if the // remote endpoint has closed the connection. For those transports, // verifyChecksum() will block until the checksum becomes available. // // Other transport types (e.g., TMemoryBuffer) always return 0 immediately // if no more data is available. For those transport types, verifyChecksum // will raise the following exception if the checksum is not available from // the underlying transport yet. throw TTransportException(TTransportException::CORRUPTED_DATA, "checksum not available yet in " "verifyChecksum()"); } // If input_ended_ is true now, the checksum has been verified if (input_ended_) { return; } // The caller invoked us before the actual end of the data stream assert(rstream_->avail_out < urbuf_size_); throw TTransportException(TTransportException::CORRUPTED_DATA, "verifyChecksum() called before end of " "zlib stream"); }
void TSaslTransport::open() { NegotiationStatus status = TSASL_INVALID; uint32_t resLength; // Only client should open the underlying transport. if (isClient_ && !transport_->isOpen()) { transport_->open(); } // initiate SASL message handleSaslStartMessage(); // SASL connection handshake while (!sasl_->isComplete()) { uint8_t* message = receiveSaslMessage(&status, &resLength); if (status == TSASL_COMPLETE) { if (isClient_) { if (!sasl_->isComplete()) { // Server sent COMPLETE out of order. throw TTransportException("Received COMPLETE but no handshake occurred"); } break; // handshake complete } } else if (status != TSASL_OK) { stringstream ss; ss << "Expected COMPLETE or OK, got " << status; throw TTransportException(ss.str()); } uint32_t challengeLength; uint8_t* challenge = sasl_->evaluateChallengeOrResponse( message, resLength, &challengeLength); sendSaslMessage(sasl_->isComplete() ? TSASL_COMPLETE : TSASL_OK, challenge, challengeLength); } // If the server isn't complete yet, we need to wait for its response. // This will occur with ANONYMOUS auth, for example, where we send an // initial response and are immediately complete. if (isClient_ && (status == TSASL_INVALID || status == TSASL_OK)) { receiveSaslMessage(&status, &resLength); if (status != TSASL_COMPLETE) { stringstream ss; ss << "Expected COMPLETE or OK, got " << status; throw TTransportException(ss.str()); } } // TODO : need to set the shouldWrap_ based on QOP /* String qop = (String) sasl.getNegotiatedProperty(Sasl.QOP); if (qop != null && !qop.equalsIgnoreCase("auth")) shouldWrap_ = true; */ }
void THDFSFileTransport::write(const uint8_t* buf, uint32_t len) { tSize rv = hdfsWrite(hdfsFile_->getFS()->getHandle(), (hdfsFile)hdfsFile_->getHandle(), buf, len); if (rv < 0) { int errno_copy = errno; throw TTransportException(TTransportException::UNKNOWN, "THDFSFileTransport::write()", errno_copy); } else if (rv != len) { throw TTransportException(TTransportException::INTERRUPTED, "THDFSFileTransport::write()"); } }
uint32_t THDFSFileTransport::read(uint8_t* buf, uint32_t len) { tSize rv = hdfsRead(hdfsFile_->getFS()->getHandle(), (hdfsFile)hdfsFile_->getHandle(), buf, len); if (rv < 0) { int errno_copy = errno; throw TTransportException(TTransportException::UNKNOWN, "THDFSFileTransport::read()", errno_copy); } else if (rv == 0) { throw TTransportException(TTransportException::END_OF_FILE, "THDFSFileTransport::read()"); } return rv; }
/** * Performs the server side of the initial portion of the Thrift SASL protocol. * Receives the initial response from the client, creates a SASL server using * the mechanism requested by the client (if this server supports it), and * sends the first challenge back to the client. */ void TSaslServerTransport::handleSaslStartMessage() { uint32_t msgLength; NegotiationStatus status; uint8_t* message = receiveSaslMessage(&status, &msgLength); if (status != TSASL_START) { stringstream ss; ss << "Expecting START status, received " << status; sendSaslMessage(TSASL_ERROR, reinterpret_cast<const uint8_t*>(ss.str().c_str()), ss.str().size()); throw TTransportException(ss.str()); } // Message is a non-null terminated string; to use it like a // C-string we have to copy it into a null-terminated buffer. // The first message should be the mechanism string. string mechanism(reinterpret_cast<char*>(message), msgLength); map<string, TSaslServerDefinition*>::iterator defn = TSaslServerTransport::serverDefinitionMap_.find(mechanism); if (defn == TSaslServerTransport::serverDefinitionMap_.end()) { stringstream ss; ss << "Unsupported mechanism type " << mechanism; sendSaslMessage(TSASL_BAD, reinterpret_cast<const uint8_t*>(ss.str().c_str()), ss.str().size()); throw TTransportException(TTransportException::BAD_ARGS, ss.str()); } TSaslServerDefinition* serverDefinition = defn->second; sasl_.reset(new TSaslServer(mechanism, serverDefinition->protocol_, serverDefinition->serverName_, serverDefinition->realm_, serverDefinition->flags_, &serverDefinition->callbacks_[0])); uint32_t challengeLength; uint8_t* challenge = sasl_->evaluateChallengeOrResponse( reinterpret_cast<const uint8_t*>(mechanism.c_str()), msgLength, &challengeLength); // TODO: this is necessary for DIGEST-MD5 but not for GSSAPI. There should be a more // general way to do this rather than checking the challengeLength. For GSSAPI, this // is zero, the server just authenticates. For DIGEST-MD5, this begins some back and // forth to send additional information. if (challengeLength != 0) { sendSaslMessage(sasl_->isComplete() ? TSASL_COMPLETE : TSASL_OK, challenge, challengeLength); } }
bool TFramedTransport::readFrame() { // TODO(dreiss): Think about using readv here, even though it would // result in (gasp) read-ahead. // Read the size of the next frame. // We can't use readAll(&sz, sizeof(sz)), since that always throws an // exception on EOF. We want to throw an exception only if EOF occurs after // partial size data. int32_t sz = -1; uint32_t size_bytes_read = 0; while (size_bytes_read < sizeof(sz)) { uint8_t* szp = reinterpret_cast<uint8_t*>(&sz) + size_bytes_read; uint32_t bytes_read = transport_->read(szp, static_cast<uint32_t>(sizeof(sz)) - size_bytes_read); if (bytes_read == 0) { if (size_bytes_read == 0) { // EOF before any data was read. return false; } else { // EOF after a partial frame header. Raise an exception. throw TTransportException(TTransportException::END_OF_FILE, "No more data to read after " "partial frame header."); } } size_bytes_read += bytes_read; } sz = ntohl(sz); if (sz < 0) { throw TTransportException("Frame size has negative value"); } // Check for oversized frame if (sz > static_cast<int32_t>(maxFrameSize_)) throw TTransportException(TTransportException::CORRUPTED_DATA, "Received an oversized frame"); // Read the frame payload, and reset markers. if (sz > static_cast<int32_t>(rBufSize_)) { rBuf_.reset(new uint8_t[sz]); rBufSize_ = sz; } transport_->readAll(rBuf_.get(), sz); setReadBuffer(rBuf_.get(), sz); return true; }
void TBufferedTransport::putBack(uint8_t* buf, uint32_t len) { assert(len < rBufSize_); // Check that len fits. // We push rBase_ back below to rBuf_.get() if (rBound_ + len > rBufSize_ + rBase_) { throw TTransportException(TTransportException::BAD_ARGS, "TBufferedTransport called with oversize buf"); } // Reset the buffer to initial position, moving any unread data. if (rBuf_.get() + len > rBase_) { if (rBase_ != rBound_) { // advance further to get room for the putback bytes memmove(rBuf_.get() + len, rBase_, rBound_ - rBase_); } rBound_ += (rBuf_.get() - rBase_); rBase_ = rBuf_.get(); } else { rBase_ -= len; rBound_ -= len; } memcpy(rBase_, buf, len); rBound_ += len; }
void TMemoryBuffer::ensureCanWrite(uint32_t len) { // Check available space uint32_t avail = available_write(); if (len <= avail) { return; } if (!owner_) { throw TTransportException("Insufficient space in external MemoryBuffer"); } // Grow the buffer as necessary. uint32_t new_size = bufferSize_; while (len > avail) { new_size = new_size > 0 ? new_size * 2 : 1; avail = available_write() + (new_size - bufferSize_); } // Allocate into a new pointer so we don't bork ours if it fails. uint8_t* new_buffer = static_cast<uint8_t*>(std::realloc(buffer_, new_size)); if (new_buffer == NULL) { throw std::bad_alloc(); } rBase_ = new_buffer + (rBase_ - buffer_); rBound_ = new_buffer + (rBound_ - buffer_); wBase_ = new_buffer + (wBase_ - buffer_); wBound_ = new_buffer + new_size; buffer_ = new_buffer; bufferSize_ = new_size; }
/** * Set the server for this transport */ void TSaslServerTransport::setSaslServer(sasl::TSasl* saslServer) { if (isClient_) { throw TTransportException( TTransportException::INTERNAL_ERROR, "Setting server in client transport"); } sasl_.reset(saslServer); }
void TQIODeviceTransport::open() { if (!isOpen()) { throw TTransportException(TTransportException::NOT_OPEN, "open(): underlying QIODevice isn't open"); } }
std::pair<folly::IOBufQueue*, size_t> ProtectionChannelHandler::decrypt(folly::IOBufQueue* q) { if (protectionState_ == ProtectionState::INVALID) { throw TTransportException("protection state is invalid"); } if (protectionState_ != ProtectionState::VALID) { // not an encrypted message, so pass-through return std::make_pair(q, 0); } assert(saslEndpoint_ != nullptr); size_t remaining = 0; if (!q->front() || q->front()->empty()) { return std::make_pair(nullptr, 0); } // decrypt unique_ptr<IOBuf> unwrapped = saslEndpoint_->unwrap(q, &remaining); assert(bool(unwrapped) ^ (remaining > 0)); // 1 and only 1 should be true if (unwrapped) { queue_.append(std::move(unwrapped)); return std::make_pair(&queue_, remaining); } else { return std::make_pair(nullptr, remaining); } }
void TFramedTransport::writeSlow(const uint8_t* buf, uint32_t len) { // Double buffer size until sufficient. uint32_t have = wBase_ - wBuf_.get(); uint32_t new_size = wBufSize_; if (len + have < have /* overflow */ || len + have > 0x7fffffff) { throw TTransportException(TTransportException::BAD_ARGS, "Attempted to write over 2 GB to TFramedTransport."); } while (new_size < len + have) { new_size = new_size > 0 ? new_size * 2 : 1; } // TODO(dreiss): Consider modifying this class to use malloc/free // so we can use realloc here. // Allocate new buffer. uint8_t* new_buf = new uint8_t[new_size]; // Copy the old buffer to the new one. memcpy(new_buf, wBuf_.get(), have); // Now point buf to the new one. wBuf_.reset(new_buf); wBufSize_ = new_size; wBase_ = wBuf_.get() + have; wBound_ = wBuf_.get() + wBufSize_; // Copy the data into the new buffer. memcpy(wBase_, buf, len); wBase_ += len; }
void TMemoryBuffer::wroteBytes(uint32_t len) { uint32_t avail = available_write(); if (len > avail) { throw TTransportException("Client wrote more bytes than size of buffer."); } wBase_ += len; }
void TZlibTransport::flush() { if (output_finished_) { throw TTransportException(TTransportException::BAD_ARGS, "flush() called after finish()"); } flushToTransport(Z_FULL_FLUSH); }
void TZlibTransport::finish() { if (output_finished_) { throw TTransportException(TTransportException::BAD_ARGS, "finish() called more than once"); } flushToTransport(Z_FINISH); }
void TZlibTransport::consume(uint32_t len) { if (readAvail() >= (int)len) { urpos_ += len; } else { throw TTransportException(TTransportException::BAD_ARGS, "consume did not follow a borrow."); } }
void THDFSFileTransport::open() { // THDFSFileTransport does not handle resource alloc/release. // An open HDFSFile handle must exist by now if (!isOpen()) { throw TTransportException(TTransportException::NOT_OPEN, "THDFSFileTransport::open()"); } }
void TFDTransport::write(const uint8_t* buf, uint32_t len) { while (len > 0) { ssize_t rv = ::write(fd_, buf, len); if (rv < 0) { int errno_copy = errno; throw TTransportException(TTransportException::UNKNOWN, "TFDTransport::write()", errno_copy); } else if (rv == 0) { throw TTransportException(TTransportException::END_OF_FILE, "TFDTransport::write()"); } buf += rv; len -= rv; } }
uint32_t TSaslTransport::readLength() { uint8_t lenBuf[PAYLOAD_LENGTH_BYTES]; transport_->readAll(lenBuf, PAYLOAD_LENGTH_BYTES); int32_t len = decodeInt(lenBuf, 0); if (len < 0) { throw TTransportException("Frame size has negative value"); } return static_cast<uint32_t>(len); }
void pipe_write(HANDLE pipe, const uint8_t* buf, uint32_t len) { DWORD cbWritten; int fSuccess = WriteFile(pipe, // pipe handle buf, // message len, // message length &cbWritten, // bytes written NULL); // not overlapped if (!fSuccess) throw TTransportException(TTransportException::NOT_OPEN, "Write to pipe failed"); }
TOverlappedSubmissionThread::TOverlappedSubmissionThread() { stopItem_.action = TOverlappedWorkItem::STOP; InitializeSListHead(&workList_); thread_ = (HANDLE)_beginthreadex(NULL, 0, thread_proc, this, 0, NULL); if (thread_ == 0) { GlobalOutput.perror("TOverlappedSubmissionThread unable to create thread, errno=", errno); throw TTransportException(TTransportException::NOT_OPEN, " TOverlappedSubmissionThread unable to create thread"); } }
uint32_t TSocket::write_partial(const uint8_t* buf, uint32_t len) { if (socket_ < 0) { throw TTransportException(TTransportException::NOT_OPEN, "Called write on non-open socket"); } uint32_t sent = 0; int flags = 0; #ifdef MSG_NOSIGNAL // Note the use of MSG_NOSIGNAL to suppress SIGPIPE errors, instead we // check for the EPIPE return condition and close the socket in that case flags |= MSG_NOSIGNAL; #endif // ifdef MSG_NOSIGNAL int b = send(socket_, buf + sent, len - sent, flags); ++g_socket_syscalls; if (b < 0) { if (errno == EWOULDBLOCK || errno == EAGAIN) { return 0; } // Fail on a send error int errno_copy = errno; GlobalOutput.perror("TSocket::write_partial() send() " + getSocketInfo(), errno_copy); if (errno_copy == EPIPE || errno_copy == ECONNRESET || errno_copy == ENOTCONN) { close(); throw TTransportException(TTransportException::NOT_OPEN, "write() send() " + getSocketInfo(), errno_copy); } throw TTransportException(TTransportException::UNKNOWN, "write() send() " + getSocketInfo(), errno_copy); } // Fail on blocked send if (b == 0) { throw TTransportException(TTransportException::NOT_OPEN, "Socket send returned 0."); } return b; }