bool zen::filesHaveSameContent(const AbstractPathRef& filePath1, const AbstractPathRef& filePath2, const std::function<void(std::int64_t bytesDelta)>& onUpdateStatus) //throw FileError { const std::unique_ptr<ABF::InputStream> inStream1 = ABF::getInputStream(filePath1); //throw FileError, (ErrorFileLocked) const std::unique_ptr<ABF::InputStream> inStream2 = ABF::getInputStream(filePath2); // BufferSize dynamicBufSize(std::min(inStream1->optimalBlockSize(), inStream2->optimalBlockSize())); TickVal lastDelayViolation = getTicks(); std::vector<char> buf; //make this thread-local? => on noticeable perf advantage! for (;;) { const size_t bufSize = dynamicBufSize.get(); //save for reliable eof check below!!! setMinSize(buf, 2 * bufSize); char* buf1 = &buf[0]; char* buf2 = &buf[bufSize]; const TickVal startTime = getTicks(); const size_t length1 = inStream1->read(buf1, bufSize); //throw FileError const size_t length2 = inStream2->read(buf2, bufSize); //returns actual number of bytes read //send progress updates immediately after reading to reliably allow speed calculations for our clients! if (onUpdateStatus) onUpdateStatus(std::max(length1, length2)); if (length1 != length2 || ::memcmp(buf1, buf2, length1) != 0) return false; //-------- dynamically set buffer size to keep callback interval between 100 - 500ms --------------------- if (TICKS_PER_SEC > 0) { const TickVal now = getTicks(); const std::int64_t loopTime = dist(startTime, now) * 1000 / TICKS_PER_SEC; //unit: [ms] if (loopTime < 100) { if (dist(lastDelayViolation, now) / TICKS_PER_SEC > 2) //avoid "flipping back": e.g. DVD-Roms read 32MB at once, so first read may be > 500 ms, but second one will be 0ms! { lastDelayViolation = now; dynamicBufSize.inc(); } } else if (loopTime > 500) { lastDelayViolation = now; dynamicBufSize.dec(); } } //------------------------------------------------------------------------------------------------ if (length1 != bufSize) //end of file return true; } }
size_t NtripStream::read(uint8_t* buffer, size_t max_length) { if (!_tcp_stream) { return 0; } size_t ret = 0; if (_tcp_stream->get_status() != Stream::Status::CONNECTED) { reconnect(); if (_status != Stream::Status::CONNECTED) { return 0; } } if (is_zero(_data_active_s)) { _data_active_s = ros::Time::now().toSec(); } ret = _tcp_stream->read(buffer, max_length); if (ret) { _data_active_s = ros::Time::now().toSec(); } // timeout detect if ((ros::Time::now().toSec() - _data_active_s) > _timeout_s) { ROS_INFO("Ntrip timeout."); reconnect(); } return ret; }
int read_columns( aku_Timestamp *timestamps , aku_ParamId *params , aku_PData *pointers , uint32_t *lengths , size_t arrays_size ) { // TODO: track PageHeader::open_count here std::vector<CursorResult> results; results.resize(arrays_size); // TODO: pass all pointers to storage directly int n_results = cursor_->read(results.data(), results.size()); for (int i = 0; i < n_results; i++) { const CursorResult& result = results[i]; if (timestamps) { timestamps[i] = result.timestamp; } if (params) { params[i] = result.param_id; } if (pointers) { pointers[i] = result.data; } if (lengths) { lengths[i] = result.length; } } return n_results; }
static std::vector<char> readDataFromFile(std::unique_ptr<std::fstream>& file) { file->seekg(0, std::ios_base::end); const std::streamsize size = file->tellg(); std::vector<char> v; v.resize(size); file->seekg(0, std::ios_base::beg); file->read(v.data(), size); return v; }
size_t FrontBufferedStream::readDirectlyFromStream(char* dst, size_t size) { SkASSERT(size > 0); // If we get here, we have buffered all that can be buffered. SkASSERT(fBufferSize == fBufferedSoFar && fOffset >= fBufferSize); const size_t bytesReadDirectly = fStream->read(dst, size); fOffset += bytesReadDirectly; // If we have read past the end of the buffer, rewinding is no longer // supported, so we can go ahead and free the memory. if (bytesReadDirectly > 0) { sk_free(fBuffer.release()); } return bytesReadDirectly; }
FileType suggestFileType( const std::unique_ptr<IFile>& file ) { if ( file == nullptr ) { LOG( "Argument 'file' is nullptr." ); return FILETYPE_UNKNOWN; } FileType suggested_type = FILETYPE_UNKNOWN; //現在位置保存 unsigned long cursor = file->tell(); //ヘッダ読み取り { char buffer[ 17 ]; buffer[ 16 ] = '\0'; //オーバーランの保険 char* str = buffer; file->seek( 0 ); file->read( str, 16 ); if ( strstr( str, "RIFF" ) == str ) { //wave str += 8; if ( strstr( str, "WAVE" ) == str ) suggested_type = FILETYPE_WAVE; } else if ( strstr( str, "OggS" ) == str ) { //ogg suggested_type = FILETYPE_OGG; } } //元の位置に戻る file->seek( cursor ); //完了 return suggested_type; }
size_t FrontBufferedStream::bufferAndWriteTo(char* dst, size_t size) { SkASSERT(size > 0); SkASSERT(fOffset >= fBufferedSoFar); SkASSERT(fBuffer); // Data needs to be buffered. Buffer up to the lesser of the size requested // and the remainder of the max buffer size. const size_t bytesToBuffer = SkTMin(size, fBufferSize - fBufferedSoFar); char* buffer = fBuffer + fOffset; const size_t buffered = fStream->read(buffer, bytesToBuffer); fBufferedSoFar += buffered; fOffset = fBufferedSoFar; SkASSERT(fBufferedSoFar <= fBufferSize); // Copy the buffer to the destination buffer and update the amount read. if (dst != nullptr) { memcpy(dst, buffer, buffered); } return buffered; }
void AttachmentReaderTest::testMultipleReads(bool closeWriterBeforeReading) { init(); auto numWritten = m_writer->write(m_testPattern.data(), m_testPattern.size()); ASSERT_EQ(numWritten, static_cast<ssize_t>(m_testPattern.size())); AttachmentReader::ReadStatus terminalStatus = AttachmentReader::ReadStatus::OK_WOULDBLOCK; if (closeWriterBeforeReading) { m_writer->close(); terminalStatus = AttachmentReader::ReadStatus::CLOSED; } std::vector<uint8_t> result(TEST_SDS_PARTIAL_READ_AMOUNT_IN_BYTES); auto readStatus = InProcessAttachmentReader::ReadStatus::OK; int totalBytesRead = 0; bool done = false; int iterations = 0; int iterationsMax = 10; while (!done && iterations < iterationsMax) { auto bytesRead = m_reader->read(result.data(), result.size(), &readStatus); if (terminalStatus == readStatus) { done = true; } for (size_t i = 0; i < bytesRead; ++i) { EXPECT_EQ(result[i], m_testPattern[i + totalBytesRead]); } totalBytesRead += bytesRead; iterations++; } // Not only was all the data read, but the reader remained open. ASSERT_NE(iterations, iterationsMax); ASSERT_EQ(readStatus, terminalStatus); ASSERT_EQ(totalBytesRead, static_cast<ssize_t>(m_testPattern.size())); }
osmium::io::Meta open() { return m_input->read(); }
int read_values( aku_Sample *values , size_t values_size ) { return cursor_->read(values, values_size); }
TransferStats SenderThread::sendOneByteSource( const std::unique_ptr<ByteSource> &source, ErrorCode transferStatus) { TransferStats stats; char headerBuf[Protocol::kMaxHeader]; int64_t off = 0; headerBuf[off++] = Protocol::FILE_CMD; headerBuf[off++] = transferStatus; char *headerLenPtr = headerBuf + off; off += sizeof(int16_t); const int64_t expectedSize = source->getSize(); int64_t actualSize = 0; const SourceMetaData &metadata = source->getMetaData(); BlockDetails blockDetails; blockDetails.fileName = metadata.relPath; blockDetails.seqId = metadata.seqId; blockDetails.fileSize = metadata.size; blockDetails.offset = source->getOffset(); blockDetails.dataSize = expectedSize; blockDetails.allocationStatus = metadata.allocationStatus; blockDetails.prevSeqId = metadata.prevSeqId; Protocol::encodeHeader(wdtParent_->getProtocolVersion(), headerBuf, off, Protocol::kMaxHeader, blockDetails); int16_t littleEndianOff = folly::Endian::little((int16_t)off); folly::storeUnaligned<int16_t>(headerLenPtr, littleEndianOff); int64_t written = socket_->write(headerBuf, off); if (written != off) { WTPLOG(ERROR) << "Write error/mismatch " << written << " " << off << ". fd = " << socket_->getFd() << ". file = " << metadata.relPath << ". port = " << socket_->getPort(); stats.setLocalErrorCode(SOCKET_WRITE_ERROR); stats.incrFailedAttempts(); return stats; } stats.addHeaderBytes(written); int64_t byteSourceHeaderBytes = written; int64_t throttlerInstanceBytes = byteSourceHeaderBytes; int64_t totalThrottlerBytes = 0; WTVLOG(3) << "Sent " << written << " on " << socket_->getFd() << " : " << folly::humanify(std::string(headerBuf, off)); int32_t checksum = 0; while (!source->finished()) { // TODO: handle protocol errors from readHeartBeats readHeartBeats(); int64_t size; char *buffer = source->read(size); if (source->hasError()) { WTLOG(ERROR) << "Failed reading file " << source->getIdentifier() << " for fd " << socket_->getFd(); break; } WDT_CHECK(buffer && size > 0); if (footerType_ == CHECKSUM_FOOTER) { checksum = folly::crc32c((const uint8_t *)buffer, size, checksum); } if (wdtParent_->getThrottler()) { /** * If throttling is enabled we call limit(deltaBytes) which * used both the methods of throttling peak and average. * Always call it with bytes being written to the wire, throttler * will do the rest. * The first time throttle is called with the header bytes * included. In the next iterations throttler is only called * with the bytes being written. */ throttlerInstanceBytes += size; wdtParent_->getThrottler()->limit(*threadCtx_, throttlerInstanceBytes); totalThrottlerBytes += throttlerInstanceBytes; throttlerInstanceBytes = 0; } written = socket_->write(buffer, size, /* retry writes */ true); if (getThreadAbortCode() != OK) { WTLOG(ERROR) << "Transfer aborted during block transfer " << socket_->getPort() << " " << source->getIdentifier(); stats.setLocalErrorCode(ABORT); stats.incrFailedAttempts(); return stats; } if (written != size) { WTLOG(ERROR) << "Write error " << written << " (" << size << ")" << ". fd = " << socket_->getFd() << ". file = " << metadata.relPath << ". port = " << socket_->getPort(); stats.setLocalErrorCode(SOCKET_WRITE_ERROR); stats.incrFailedAttempts(); return stats; } stats.addDataBytes(written); actualSize += written; } if (actualSize != expectedSize) { // Can only happen if sender thread can not read complete source byte // stream WTLOG(ERROR) << "UGH " << source->getIdentifier() << " " << expectedSize << " " << actualSize; struct stat fileStat; if (stat(metadata.fullPath.c_str(), &fileStat) != 0) { WTPLOG(ERROR) << "stat failed on path " << metadata.fullPath; } else { WTLOG(WARNING) << "file " << source->getIdentifier() << " previous size " << metadata.size << " current size " << fileStat.st_size; } stats.setLocalErrorCode(BYTE_SOURCE_READ_ERROR); stats.incrFailedAttempts(); return stats; } if (wdtParent_->getThrottler() && actualSize > 0) { WDT_CHECK(totalThrottlerBytes == actualSize + byteSourceHeaderBytes) << totalThrottlerBytes << " " << (actualSize + totalThrottlerBytes); } if (footerType_ != NO_FOOTER) { off = 0; headerBuf[off++] = Protocol::FOOTER_CMD; Protocol::encodeFooter(headerBuf, off, Protocol::kMaxFooter, checksum); int toWrite = off; written = socket_->write(headerBuf, toWrite); if (written != toWrite) { WTLOG(ERROR) << "Write mismatch " << written << " " << toWrite; stats.setLocalErrorCode(SOCKET_WRITE_ERROR); stats.incrFailedAttempts(); return stats; } stats.addHeaderBytes(toWrite); } stats.setLocalErrorCode(OK); stats.incrNumBlocks(); stats.addEffectiveBytes(stats.getHeaderBytes(), stats.getDataBytes()); return stats; }
/** * Read implementation delegated to the underlying source * * @param buffer output buffer * @param length number of bytes to process * @return number of bytes processed */ std::streamsize read(char* buffer, std::streamsize length) { return src->read(buffer, length); }
osmium::io::Header open(osmium::item_flags_type read_types = osmium::item_flags_type::all) { m_read_types = read_types; return m_input->read(read_types); }
std::unique_ptr<SkCodec> SkIcoCodec::MakeFromStream(std::unique_ptr<SkStream> stream, Result* result) { // Header size constants constexpr uint32_t kIcoDirectoryBytes = 6; constexpr uint32_t kIcoDirEntryBytes = 16; // Read the directory header std::unique_ptr<uint8_t[]> dirBuffer(new uint8_t[kIcoDirectoryBytes]); if (stream->read(dirBuffer.get(), kIcoDirectoryBytes) != kIcoDirectoryBytes) { SkCodecPrintf("Error: unable to read ico directory header.\n"); *result = kIncompleteInput; return nullptr; } // Process the directory header const uint16_t numImages = get_short(dirBuffer.get(), 4); if (0 == numImages) { SkCodecPrintf("Error: No images embedded in ico.\n"); *result = kInvalidInput; return nullptr; } // This structure is used to represent the vital information about entries // in the directory header. We will obtain this information for each // directory entry. struct Entry { uint32_t offset; uint32_t size; }; SkAutoFree dirEntryBuffer(sk_malloc_canfail(sizeof(Entry) * numImages)); if (!dirEntryBuffer) { SkCodecPrintf("Error: OOM allocating ICO directory for %i images.\n", numImages); *result = kInternalError; return nullptr; } auto* directoryEntries = reinterpret_cast<Entry*>(dirEntryBuffer.get()); // Iterate over directory entries for (uint32_t i = 0; i < numImages; i++) { uint8_t entryBuffer[kIcoDirEntryBytes]; if (stream->read(entryBuffer, kIcoDirEntryBytes) != kIcoDirEntryBytes) { SkCodecPrintf("Error: Dir entries truncated in ico.\n"); *result = kIncompleteInput; return nullptr; } // The directory entry contains information such as width, height, // bits per pixel, and number of colors in the color palette. We will // ignore these fields since they are repeated in the header of the // embedded image. In the event of an inconsistency, we would always // defer to the value in the embedded header anyway. // Specifies the size of the embedded image, including the header uint32_t size = get_int(entryBuffer, 8); // Specifies the offset of the embedded image from the start of file. // It does not indicate the start of the pixel data, but rather the // start of the embedded image header. uint32_t offset = get_int(entryBuffer, 12); // Save the vital fields directoryEntries[i].offset = offset; directoryEntries[i].size = size; } // Default Result, if no valid embedded codecs are found. *result = kInvalidInput; // It is "customary" that the embedded images will be stored in order of // increasing offset. However, the specification does not indicate that // they must be stored in this order, so we will not trust that this is the // case. Here we sort the embedded images by increasing offset. struct EntryLessThan { bool operator() (Entry a, Entry b) const { return a.offset < b.offset; } }; EntryLessThan lessThan; SkTQSort(directoryEntries, &directoryEntries[numImages - 1], lessThan); // Now will construct a candidate codec for each of the embedded images uint32_t bytesRead = kIcoDirectoryBytes + numImages * kIcoDirEntryBytes; std::unique_ptr<SkTArray<std::unique_ptr<SkCodec>, true>> codecs( new SkTArray<std::unique_ptr<SkCodec>, true>(numImages)); for (uint32_t i = 0; i < numImages; i++) { uint32_t offset = directoryEntries[i].offset; uint32_t size = directoryEntries[i].size; // Ensure that the offset is valid if (offset < bytesRead) { SkCodecPrintf("Warning: invalid ico offset.\n"); continue; } // If we cannot skip, assume we have reached the end of the stream and // stop trying to make codecs if (stream->skip(offset - bytesRead) != offset - bytesRead) { SkCodecPrintf("Warning: could not skip to ico offset.\n"); break; } bytesRead = offset; // Create a new stream for the embedded codec SkAutoFree buffer(sk_malloc_canfail(size)); if (!buffer) { SkCodecPrintf("Warning: OOM trying to create embedded stream.\n"); break; } if (stream->read(buffer.get(), size) != size) { SkCodecPrintf("Warning: could not create embedded stream.\n"); *result = kIncompleteInput; break; } sk_sp<SkData> data(SkData::MakeFromMalloc(buffer.release(), size)); auto embeddedStream = SkMemoryStream::Make(data); bytesRead += size; // Check if the embedded codec is bmp or png and create the codec std::unique_ptr<SkCodec> codec; Result dummyResult; if (SkPngCodec::IsPng((const char*) data->bytes(), data->size())) { codec = SkPngCodec::MakeFromStream(std::move(embeddedStream), &dummyResult); } else { codec = SkBmpCodec::MakeFromIco(std::move(embeddedStream), &dummyResult); } // Save a valid codec if (nullptr != codec) { codecs->push_back().reset(codec.release()); } } // Recognize if there are no valid codecs if (0 == codecs->count()) { SkCodecPrintf("Error: could not find any valid embedded ico codecs.\n"); return nullptr; } // Use the largest codec as a "suggestion" for image info size_t maxSize = 0; int maxIndex = 0; for (int i = 0; i < codecs->count(); i++) { SkImageInfo info = codecs->operator[](i)->getInfo(); size_t size = info.computeMinByteSize(); if (size > maxSize) { maxSize = size; maxIndex = i; } } auto maxInfo = codecs->operator[](maxIndex)->getEncodedInfo().copy(); *result = kSuccess; // The original stream is no longer needed, because the embedded codecs own their // own streams. return std::unique_ptr<SkCodec>(new SkIcoCodec(std::move(maxInfo), codecs.release())); }
bool NtripStream::connect() { if (_is_login) { return true; } if (!_tcp_stream) { ROS_ERROR("New tcp stream failed."); return true; } if (!_tcp_stream->connect()) { _status = Stream::Status::DISCONNECTED; ROS_ERROR("Tcp connect failed."); return false; } uint8_t buffer[2048]; size_t size = 0; size_t try_times = 0; size = _tcp_stream->write( reinterpret_cast<const uint8_t*>(_login_data.data()), _login_data.size()); if (size != _login_data.size()) { _tcp_stream->disconnect(); _status = Stream::Status::ERROR; ROS_ERROR("Send ntrip request failed."); return false; } bzero(buffer, sizeof(buffer)); ROS_INFO("Read ntrip response."); size = _tcp_stream->read(buffer, sizeof(buffer) - 1); while ((size == 0) && (try_times < 3)) { sleep(1); size = _tcp_stream->read(buffer, sizeof(buffer) - 1); ++try_times; } if (!size) { _tcp_stream->disconnect(); _status = Stream::Status::DISCONNECTED; ROS_ERROR("No response from ntripcaster."); return false; } if (std::strstr(reinterpret_cast<char*>(buffer), "ICY 200 OK\r\n")) { _status = Stream::Status::CONNECTED; _is_login = true; ROS_INFO("Ntrip login successfully."); return true; } if (std::strstr(reinterpret_cast<char*>(buffer), "SOURCETABLE 200 OK\r\n")) { ROS_ERROR_STREAM("Mountpoint " << _mountpoint << " not exist."); } if (std::strstr(reinterpret_cast<char*>(buffer), "HTTP/")) { ROS_ERROR("Authentication failed."); } ROS_INFO_STREAM("No expect data."); ROS_INFO_STREAM("Recv data length: " << size); // ROS_INFO_STREAM("Data from server: " << reinterpret_cast<char*>(buffer)); _tcp_stream->disconnect(); _status = Stream::Status::ERROR; return false; }
REQUIRE( tiff_reader.is_tiled() == false ); REQUIRE( tiff_reader.tile_width() == 0 ); REQUIRE( tiff_reader.tile_height() == 0 ); REQUIRE( tiff_reader.photometric() == PHOTOMETRIC_PALETTE ); REQUIRE( tiff_reader.compression() == COMPRESSION_NONE ); std::unique_ptr<mapnik::image_reader> reader(mapnik::get_image_reader(filename,"tiff")); REQUIRE( reader->width() == 512 ); REQUIRE( reader->height() == 512 ); mapnik::util::file file(filename); mapnik::tiff_reader<boost::iostreams::array_source> tiff_reader2(file.data().get(),file.size()); REQUIRE( tiff_reader2.width() == 512 ); REQUIRE( tiff_reader2.height() == 512 ); std::unique_ptr<mapnik::image_reader> reader2(mapnik::get_image_reader(file.data().get(),file.size())); REQUIRE( reader2->width() == 512 ); REQUIRE( reader2->height() == 512 ); mapnik::image_any data = reader->read(0, 0, reader->width(), reader->height()); REQUIRE( data.is<mapnik::image_rgba8>() == true ); TIFF_ASSERT_SIZE( data,reader ); TIFF_ASSERT_NO_ALPHA_RGB( data ); TIFF_READ_ONE_PIXEL } SECTION("scan rgb8 tiled") { std::string filename("./test-data/tiff/scan_512x512_rgb8_tiled.tif"); mapnik::tiff_reader<boost::iostreams::file_source> tiff_reader(filename); REQUIRE( tiff_reader.width() == 512 ); REQUIRE( tiff_reader.height() == 512 ); REQUIRE( tiff_reader.planar_config() == PLANARCONFIG_CONTIG ); REQUIRE( tiff_reader.rows_per_strip() == 0 ); REQUIRE( tiff_reader.bits_per_sample() == 8 ); REQUIRE( tiff_reader.is_tiled() == true );
int main(int argc, char** argv) { std::cout << "Image from string..." << std::endl; if (argc!=3) { std::cout << "Usage:" << argv[0] << " <path-to-image-file> <num_runs>" << std::endl; return 1; } unsigned NUM_RUNS = std::atoi(argv[2]); std::string filename(argv[1]); // node-blend { std::cerr << "========== Node-blend ImageReader FROM FILE" << std::endl; std::cout << "NUM_RUNS="<<NUM_RUNS << std::endl; boost::timer::auto_cpu_timer t; for (unsigned count=0; count < NUM_RUNS; ++count) { std::ifstream is(filename.c_str() , std::ios::binary); std::string buffer((std::istreambuf_iterator<char>(is)), std::istreambuf_iterator<char>()); const std::unique_ptr<ImageReader> layer(ImageReader::create((uint8_t*)buffer.data(),buffer.size())); layer->decode(); } } { // mapnik std::cerr << "========== Mapnik image_reader FROM FILE" << std::endl; std::cout << "NUM_RUNS="<<NUM_RUNS << std::endl; boost::timer::auto_cpu_timer t; for (unsigned count=0; count < NUM_RUNS; ++count) { try { const std::unique_ptr<mapnik::image_reader> reader(mapnik::get_image_reader(filename)); unsigned width = reader->width(); unsigned height = reader->height(); mapnik::image_data_32 buffer(width,height); reader->read(0,0,buffer); } catch (mapnik::image_reader_exception const& ex) { std::cerr << ex.what() << std::endl; } catch( std::exception const& ex) { std::cerr << ex.what() << std::endl; } } } std::ifstream is(filename.c_str() , std::ios::binary); std::string buffer((std::istreambuf_iterator<char>(is)), std::istreambuf_iterator<char>()); // node-blend { std::cerr << "========== Node-blend ImageReader FROM MEM BUFFER" << std::endl; std::cout << "NUM_RUNS="<<NUM_RUNS << std::endl; boost::timer::auto_cpu_timer t; for (unsigned count=0; count < NUM_RUNS; ++count) { const std::unique_ptr<ImageReader> layer(ImageReader::create((uint8_t*)buffer.data(),buffer.size())); layer->decode(); } } { // mapnik std::cerr << "========== Mapnik image_reader FROM MEM BUFFER" << std::endl; std::cout << "NUM_RUNS="<<NUM_RUNS << std::endl; boost::timer::auto_cpu_timer t; for (unsigned count=0; count < NUM_RUNS; ++count) { try { const std::unique_ptr<mapnik::image_reader> reader(mapnik::get_image_reader(buffer.data(), buffer.size())); unsigned width = reader->width(); unsigned height = reader->height(); mapnik::image_data_32 buffer(width,height); reader->read(0,0,buffer); } catch (mapnik::image_reader_exception const& ex) { std::cerr << ex.what() << std::endl; } catch( std::exception const& ex) { std::cerr << ex.what() << std::endl; } } } return 0; }