int
syncindex(Index *ix)
{
Arena *arena;
int i, e, e1, ok;
ok = 0;
for(i = 0; i < ix->narenas; i++){
trace(TraceProc, "syncindex start %d", i);
arena = ix->arenas[i];
e = syncarena(arena, TWID32, 1, 1);
e1 = e;
e1 &= ~(SyncHeader|SyncCIZero|SyncCIErr);
if(e & SyncHeader)
fprint(2, "arena %s: header is out-of-date\n", arena->name);
if(e1){
fprint(2, "arena %s: %x\n", arena->name, e1);
ok = -1;
continue;
}
flushdcache();
if(arena->memstats.clumps == arena->diskstats.clumps)
continue;
fprint(2, "%T %s: indexing %d clumps...\n",
arena->name,
arena->memstats.clumps - arena->diskstats.clumps);
if(syncarenaindex(arena, ix->amap[i].start) < 0){
fprint(2, "arena %s: syncarenaindex: %r\n", arena->name);
ok = -1;
continue;
}
if(wbarena(arena) < 0){
fprint(2, "arena %s: wbarena: %r\n", arena->name);
ok = -1;
continue;
}
flushdcache();
delaykickicache();
}
return ok;
}
Arena*
newarena(Part *part, uint32_t vers, char *name, uint64_t base,
uint64_t size,
uint32_t blocksize)
{
int bsize;
Arena *arena;
if(nameok(name) < 0){
seterr(EOk, "illegal arena name", name);
return nil;
}
arena = MKZ(Arena);
arena->part = part;
arena->version = vers;
if(vers == ArenaVersion4)
arena->clumpmagic = _ClumpMagic;
else{
do
arena->clumpmagic = fastrand();
while(arena->clumpmagic==_ClumpMagic || arena->clumpmagic==0);
}
arena->blocksize = blocksize;
arena->clumpmax = arena->blocksize / ClumpInfoSize;
arena->base = base + blocksize;
arena->size = size - 2 * blocksize;
namecp(arena->name, name);
bsize = sizeof zero;
if(bsize > arena->blocksize)
bsize = arena->blocksize;
if(wbarena(arena)<0 || wbarenahead(arena)<0
|| writepart(arena->part, arena->base, zero, bsize)<0){
freearena(arena);
return nil;
}
return arena;
}
void
setatailstate(AState *as)
{
int i, j, osealed;
Arena *a;
Index *ix;
trace(0, "setatailstate %s 0x%llux clumps %d", as->arena->name, as->aa, as->stats.clumps);
/*
* Look up as->arena to find index.
*/
needmainindex(); /* OS X linker */
ix = mainindex;
for(i=0; i<ix->narenas; i++)
if(ix->arenas[i] == as->arena)
break;
if(i==ix->narenas || as->aa < ix->amap[i].start || as->aa >= ix->amap[i].stop || as->arena != ix->arenas[i]){
fprint(2, "funny settailstate 0x%llux\n", as->aa);
return;
}
for(j=0; j<=i; j++){
a = ix->arenas[j];
if(atailcmp(&a->diskstats, &a->memstats) == 0)
continue;
qlock(&a->lock);
osealed = a->diskstats.sealed;
if(j == i)
a->diskstats = as->stats;
else
a->diskstats = a->memstats;
wbarena(a);
if(a->diskstats.sealed != osealed && !a->inqueue)
sealarena(a);
qunlock(&a->lock);
}
}
void
sumarena(Arena *arena)
{
ZBlock *b;
DigestState s;
uint64_t a, e;
uint32_t bs;
int t;
uint8_t score[VtScoreSize];
bs = MaxIoSize;
if(bs < arena->blocksize)
bs = arena->blocksize;
/*
* read & sum all blocks except the last one
*/
flushdcache();
memset(&s, 0, sizeof s);
b = alloczblock(bs, 0, arena->part->blocksize);
e = arena->base + arena->size;
for(a = arena->base - arena->blocksize; a + arena->blocksize <= e; a += bs){
disksched();
while((t=arenasumsleeptime) == SleepForever){
sleep(1000);
disksched();
}
sleep(t);
if(a + bs > e)
bs = arena->blocksize;
if(readpart(arena->part, a, b->data, bs) < 0)
goto ReadErr;
addstat(StatSumRead, 1);
addstat(StatSumReadBytes, bs);
sha1(b->data, bs, nil, &s);
}
/*
* the last one is special, since it may already have the checksum included
*/
bs = arena->blocksize;
if(readpart(arena->part, e, b->data, bs) < 0){
ReadErr:
logerr(EOk, "sumarena can't sum %s, read at %lld failed: %r", arena->name, a);
freezblock(b);
return;
}
addstat(StatSumRead, 1);
addstat(StatSumReadBytes, bs);
sha1(b->data, bs-VtScoreSize, nil, &s);
sha1(zeroscore, VtScoreSize, nil, &s);
sha1(nil, 0, score, &s);
/*
* check for no checksum or the same
*/
if(scorecmp(score, &b->data[bs - VtScoreSize]) != 0
&& scorecmp(zeroscore, &b->data[bs - VtScoreSize]) != 0)
logerr(EOk, "overwriting mismatched checksums for arena=%s, found=%V calculated=%V",
arena->name, &b->data[bs - VtScoreSize], score);
freezblock(b);
qlock(&arena->lock);
scorecp(arena->score, score);
wbarena(arena);
qunlock(&arena->lock);
}
/*
* allocate space for the clump and write it,
* updating the arena directory
ZZZ question: should this distinguish between an arena
filling up and real errors writing the clump?
*/
uint64_t
writeaclump(Arena *arena, Clump *c, uint8_t *clbuf)
{
DBlock *b;
uint64_t a, aa;
uint32_t clump, n, nn, m, off, blocksize;
int ok;
n = c->info.size + ClumpSize + U32Size;
qlock(&arena->lock);
aa = arena->memstats.used;
if(arena->memstats.sealed
|| aa + n + U32Size + arenadirsize(arena, arena->memstats.clumps + 1) > arena->size){
if(!arena->memstats.sealed){
logerr(EOk, "seal memstats %s", arena->name);
arena->memstats.sealed = 1;
wbarena(arena);
}
qunlock(&arena->lock);
return TWID64;
}
if(packclump(c, &clbuf[0], arena->clumpmagic) < 0){
qunlock(&arena->lock);
return TWID64;
}
/*
* write the data out one block at a time
*/
blocksize = arena->blocksize;
a = arena->base + aa;
off = a & (blocksize - 1);
a -= off;
nn = 0;
for(;;){
b = getdblock(arena->part, a, off != 0 ? ORDWR : OWRITE);
if(b == nil){
qunlock(&arena->lock);
return TWID64;
}
dirtydblock(b, DirtyArena);
m = blocksize - off;
if(m > n - nn)
m = n - nn;
memmove(&b->data[off], &clbuf[nn], m);
ok = 0;
putdblock(b);
if(ok < 0){
qunlock(&arena->lock);
return TWID64;
}
nn += m;
if(nn == n)
break;
off = 0;
a += blocksize;
}
arena->memstats.used += c->info.size + ClumpSize;
arena->memstats.uncsize += c->info.uncsize;
if(c->info.size < c->info.uncsize)
arena->memstats.cclumps++;
clump = arena->memstats.clumps;
if(clump % ArenaCIGSize == 0){
if(arena->cig == nil){
loadcig(arena);
if(arena->cig == nil)
goto NoCIG;
}
/* add aa as start of next cig */
if(clump/ArenaCIGSize != arena->ncig){
fprint(2, "bad arena cig computation %s: writing clump %d but %d cigs\n",
arena->name, clump, arena->ncig);
arena->ncig = -1;
vtfree(arena->cig);
arena->cig = nil;
goto NoCIG;
}
arena->cig = vtrealloc(arena->cig, (arena->ncig+1)*sizeof arena->cig[0]);
arena->cig[arena->ncig++].offset = aa;
}
NoCIG:
arena->memstats.clumps++;
if(arena->memstats.clumps == 0)
sysfatal("clumps wrapped");
arena->wtime = now();
if(arena->ctime == 0)
arena->ctime = arena->wtime;
writeclumpinfo(arena, clump, &c->info);
wbarena(arena);
qunlock(&arena->lock);
return aa;
}
