//-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------//
// block read from handle
size_t libcore::blk_read(int handle, off64_t offset, void *pdata, size_t size)
{
if (offset != off64_t(-1))
{
int rc = ::lseek64(handle, offset, SEEK_SET);
if (rc == -1) return -1;
}
uint8_t *p = (uint8_t *)pdata;
size_t cur_size = size;
for (;;)
{
if (cur_size == 0) break;
ssize_t rc = ::read(handle, p, cur_size);
if (rc == 0)
{
return -1;
}
if (rc == -1)
{
if (errno == EAGAIN)
{
continue;
}
return -1;
}
p += rc;
cur_size -= rc;
}
return size;
}
//-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------//
// block write to handle
size_t libcore::blk_write(int handle, off64_t offset, const void *pdata, size_t size, bool flag_sync)
{
if (offset != off64_t(-1))
{
int rc = ::lseek64(handle, offset, SEEK_SET);
if (rc == -1) return -1;
}
uint8_t *p = (uint8_t *)pdata;
size_t cur_size = size;
for (;;)
{
size_t rc = ::write(handle, p, cur_size);
if (rc == size_t(-1)) return -1;
p += rc;
cur_size -= rc;
if (cur_size == 0) break;
}
// fsync file
if (flag_sync != false)
{
int rc = ::fdatasync(handle);
if (rc == -1)
{
return -1;
}
}
return size;
}
int ff_read(const char *ffpath, ffReadCallbackFunc callback, void *userData)
{
// open the flat file
int fffd = open(ffpath, O_RDONLY | O_LARGEFILE);
if (fffd == -1)
errexit("ff_read: could not open %s, %s\n", ffpath,
khstrerror(errno).c_str());
struct stat64 sb;
fstat64(fffd, &sb);
// use a 100MB window into the file for mapping
const size_t windowSize = 100*1024*1024;
const long pageSize = sysconf(_SC_PAGESIZE);
const long pageOffMask = (pageSize - 1);
const long pageMask = ~pageOffMask;
off64_t ffoff = 0; // current offset into flatfile
uint64 recnum = 0;
uint64 winnum = 0;
void* mapAddr = 0;
while (ffoff < sb.st_size) {
assert((ffoff & 31) == 0);
#if __ia64__
uint64 ffpgoff = ffoff & pageMask;
size_t ffwin; // end of the current window
ffwin = ffpgoff + windowSize;
if ((int)ffwin > sb.st_size)
#else
off64_t ffpgoff = ffoff & pageMask;
off64_t ffwin = ffpgoff + windowSize; // end of the current window
if ( ffwin > sb.st_size )
#endif
ffwin = sb.st_size;
mapAddr = mmap64(mapAddr, windowSize, PROT_READ,
(mapAddr == 0 ? 0 : MAP_FIXED) | MAP_SHARED,
fffd, ffpgoff);
if (mapAddr == MAP_FAILED)
errexit("ff_read: could not mmap flatfile %s, %s\n", ffpath,
khstrerror(errno).c_str());
const char* ffp = (const char*)mapAddr + (ffoff & pageOffMask);
while (ffoff < (int)ffwin) {
// get header from bigendian buffer
// the data is 32bit aligned so casting buffer is safe
FFRecHeader header = *(const FFRecHeader*)ffp;
header.BigEndianToHost();
// Todo: Don't use literal '24' here
if (header.len() > 1024*1024*3 || header.level() > 24 ||
header.x() > (1ull << header.level()) ||
header.y() > (1ull << header.level())) {
fprintf(stderr, "Suspicious ff file - possibly invalid record in:\n");
fprintf(stderr, "tile [%lu] len %d - %d %d %d - vers %d\n",
(unsigned long int)recnum,
(int)header.len(), (int)header.level(),
(int)header.x(), (int)header.y(),
(int)header.vers());
munmap(mapAddr, windowSize);
close(fffd);
return -1;
}
off64_t clen = header.paddedLen();
// check for premature end of file
if (ffoff + sizeof(FFRecHeader) + clen > (unsigned int)sb.st_size) {
fprintf(stderr, "Broken ff file - end of file in:\n");
fprintf(stderr, "tile [%lu] len %d - %d %d %d - vers %d\n",
(unsigned long int)recnum,
(int)header.len(), (int)header.level(),
(int)header.x(), (int)header.y(),
(int)header.vers());
munmap(mapAddr, windowSize);
close(fffd);
return -1;
}
// remap the window if the compressed data spans current
if (off64_t(ffoff + sizeof(FFRecHeader) + clen) > ffwin)
break;
const char* jp = ffp + sizeof(FFRecHeader);
if (!(*callback)(&header, (void *) jp, userData)) {
// the callback has requested that we stop traversing
munmap(mapAddr, windowSize);
close(fffd);
return 0;
}
ffoff += sizeof(FFRecHeader) + clen;
ffp += sizeof(FFRecHeader) + clen;
recnum++;
} // while ffoff < ffwin
winnum ++;
} // while ffoff < sb.st_size
munmap(mapAddr, windowSize);
close(fffd);
return 0;
} // End ff_read()
