static void buildAllBlocks()
{
auto e = mapVec.end();
auto i = mapVec.begin();
while(i!=e) {
const Map *map = gCurMap = &(*(i++));
const uint8_t *end = map->size + map->p;
const uint8_t *p = map->p;
startMap(p);
while(1) {
if(unlikely(end<=p)) break;
bool done = buildBlock(p, end);
if(done) break;
}
endMap(p);
}
}
static void buildBlockHeaders() {
info("pass 1 -- walk all blocks and build headers ...");
size_t nbBlocks = 0;
size_t baseOffset = 0;
size_t earlyMissCnt = 0;
uint8_t buf[8+gHeaderSize];
const auto sz = sizeof(buf);
const auto startTime = usecs();
const auto oneMeg = 1024 * 1024;
for(const auto &map : mapVec) {
startMap(0);
while(1) {
auto nbRead = read(map.fd, buf, sz);
if(nbRead<(signed)sz) {
break;
}
startBlock((uint8_t*)0);
uint8_t *hash = 0;
Block *prevBlock = 0;
size_t blockSize = 0;
getBlockHeader(blockSize, prevBlock, hash, earlyMissCnt, buf);
if(unlikely(0==hash)) {
break;
}
auto where = lseek(map.fd, (blockSize + 8) - sz, SEEK_CUR);
auto blockOffset = where - blockSize;
if(where<0) {
break;
}
auto block = Block::alloc();
block->init(hash, &map, blockSize, prevBlock, blockOffset);
gBlockMap[hash] = block;
endBlock((uint8_t*)0);
++nbBlocks;
}
baseOffset += map.size;
auto now = usecs();
auto elapsed = now - startTime;
auto bytesPerSec = baseOffset / (elapsed*1e-6);
auto bytesLeft = gChainSize - baseOffset;
auto secsLeft = bytesLeft / bytesPerSec;
fprintf(
stderr,
"%.2f%% (%.2f/%.2f Gigs) -- %6d blocks -- %.2f Megs/sec -- ETA %.0f secs -- ELAPSED %.0f secs \r",
(100.0*baseOffset)/gChainSize,
baseOffset/(1000.0*oneMeg),
gChainSize/(1000.0*oneMeg),
(int)nbBlocks,
bytesPerSec*1e-6,
secsLeft,
elapsed*1e-6
);
fflush(stderr);
endMap(0);
}
if(0==nbBlocks) {
warning("found no blocks - giving up");
exit(1);
}
char msg[128];
msg[0] = 0;
if(0<earlyMissCnt) {
sprintf(msg, ", %d early link misses", (int)earlyMissCnt);
}
auto elapsed = 1e-6*(usecs() - startTime);
info(
"pass 1 -- took %.0f secs, %6d blocks, %.2f Gigs, %.2f Megs/secs %s ",
elapsed,
(int)nbBlocks,
(gChainSize * 1e-9),
(gChainSize * 1e-6) / elapsed,
msg
);
}
