//-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------// // block read from handle size_t libcore::blk_read(int handle, off64_t offset, void *pdata, size_t size) { if (offset != off64_t(-1)) { int rc = ::lseek64(handle, offset, SEEK_SET); if (rc == -1) return -1; } uint8_t *p = (uint8_t *)pdata; size_t cur_size = size; for (;;) { if (cur_size == 0) break; ssize_t rc = ::read(handle, p, cur_size); if (rc == 0) { return -1; } if (rc == -1) { if (errno == EAGAIN) { continue; } return -1; } p += rc; cur_size -= rc; } return size; }
//-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------// // block write to handle size_t libcore::blk_write(int handle, off64_t offset, const void *pdata, size_t size, bool flag_sync) { if (offset != off64_t(-1)) { int rc = ::lseek64(handle, offset, SEEK_SET); if (rc == -1) return -1; } uint8_t *p = (uint8_t *)pdata; size_t cur_size = size; for (;;) { size_t rc = ::write(handle, p, cur_size); if (rc == size_t(-1)) return -1; p += rc; cur_size -= rc; if (cur_size == 0) break; } // fsync file if (flag_sync != false) { int rc = ::fdatasync(handle); if (rc == -1) { return -1; } } return size; }
int ff_read(const char *ffpath, ffReadCallbackFunc callback, void *userData) { // open the flat file int fffd = open(ffpath, O_RDONLY | O_LARGEFILE); if (fffd == -1) errexit("ff_read: could not open %s, %s\n", ffpath, khstrerror(errno).c_str()); struct stat64 sb; fstat64(fffd, &sb); // use a 100MB window into the file for mapping const size_t windowSize = 100*1024*1024; const long pageSize = sysconf(_SC_PAGESIZE); const long pageOffMask = (pageSize - 1); const long pageMask = ~pageOffMask; off64_t ffoff = 0; // current offset into flatfile uint64 recnum = 0; uint64 winnum = 0; void* mapAddr = 0; while (ffoff < sb.st_size) { assert((ffoff & 31) == 0); #if __ia64__ uint64 ffpgoff = ffoff & pageMask; size_t ffwin; // end of the current window ffwin = ffpgoff + windowSize; if ((int)ffwin > sb.st_size) #else off64_t ffpgoff = ffoff & pageMask; off64_t ffwin = ffpgoff + windowSize; // end of the current window if ( ffwin > sb.st_size ) #endif ffwin = sb.st_size; mapAddr = mmap64(mapAddr, windowSize, PROT_READ, (mapAddr == 0 ? 0 : MAP_FIXED) | MAP_SHARED, fffd, ffpgoff); if (mapAddr == MAP_FAILED) errexit("ff_read: could not mmap flatfile %s, %s\n", ffpath, khstrerror(errno).c_str()); const char* ffp = (const char*)mapAddr + (ffoff & pageOffMask); while (ffoff < (int)ffwin) { // get header from bigendian buffer // the data is 32bit aligned so casting buffer is safe FFRecHeader header = *(const FFRecHeader*)ffp; header.BigEndianToHost(); // Todo: Don't use literal '24' here if (header.len() > 1024*1024*3 || header.level() > 24 || header.x() > (1ull << header.level()) || header.y() > (1ull << header.level())) { fprintf(stderr, "Suspicious ff file - possibly invalid record in:\n"); fprintf(stderr, "tile [%lu] len %d - %d %d %d - vers %d\n", (unsigned long int)recnum, (int)header.len(), (int)header.level(), (int)header.x(), (int)header.y(), (int)header.vers()); munmap(mapAddr, windowSize); close(fffd); return -1; } off64_t clen = header.paddedLen(); // check for premature end of file if (ffoff + sizeof(FFRecHeader) + clen > (unsigned int)sb.st_size) { fprintf(stderr, "Broken ff file - end of file in:\n"); fprintf(stderr, "tile [%lu] len %d - %d %d %d - vers %d\n", (unsigned long int)recnum, (int)header.len(), (int)header.level(), (int)header.x(), (int)header.y(), (int)header.vers()); munmap(mapAddr, windowSize); close(fffd); return -1; } // remap the window if the compressed data spans current if (off64_t(ffoff + sizeof(FFRecHeader) + clen) > ffwin) break; const char* jp = ffp + sizeof(FFRecHeader); if (!(*callback)(&header, (void *) jp, userData)) { // the callback has requested that we stop traversing munmap(mapAddr, windowSize); close(fffd); return 0; } ffoff += sizeof(FFRecHeader) + clen; ffp += sizeof(FFRecHeader) + clen; recnum++; } // while ffoff < ffwin winnum ++; } // while ffoff < sb.st_size munmap(mapAddr, windowSize); close(fffd); return 0; } // End ff_read()