/* * initialize the external bucket sorting data structures */ static IEBucks* initiebucks(Part *part, int bits, uint32_t size) { IEBucks *ib; int i; ib = MKZ(IEBucks); if(ib == nil){ seterr(EOk, "out of memory"); return nil; } ib->bits = bits; ib->nbucks = 1 << bits; ib->size = size; ib->usable = (size - U32Size) / IEntrySize * IEntrySize; ib->bucks = MKNZ(IEBuck, ib->nbucks); if(ib->bucks == nil){ seterr(EOk, "out of memory allocation sorting buckets"); freeiebucks(ib); return nil; } ib->xbuf = MKN(uint8_t, size * ((1 << bits)+1)); ib->buf = (uint8_t*)(((uintptr)ib->xbuf+size-1)&~(uintptr)(size-1)); if(ib->buf == nil){ seterr(EOk, "out of memory allocating sorting buckets' buffers"); freeiebucks(ib); return nil; } for(i = 0; i < ib->nbucks; i++){ ib->bucks[i].head = TWID32; ib->bucks[i].buf = &ib->buf[i * size]; } ib->part = part; return ib; }
/* * finish sorting: * for each bucket, read it in and sort it * write out the the final file */ static uint64_t sortiebucks(IEBucks *ib) { uint64_t tot; uint32_t n; int i; if(flushiebucks(ib) < 0) return TWID64; for(i = 0; i < ib->nbucks; i++) ib->bucks[i].buf = nil; ib->off = (uint64_t)ib->chunks * ib->size; free(ib->xbuf); ib->buf = MKN(uint8_t, ib->max + U32Size); if(ib->buf == nil){ seterr(EOk, "out of memory allocating final sorting buffer; try more buckets"); return TWID64; } tot = 0; for(i = 0; i < ib->nbucks; i++){ n = sortiebuck(ib, i); if(n == TWID32) return TWID64; if(n != ib->bucks[i].total/IEntrySize) fprint(2, "bucket %d changed count %d => %d\n", i, (int)(ib->bucks[i].total/IEntrySize), n); tot += n; } return tot; }
void bjobcode(void) { struct symtab *sp; struct rstack *rp; NODE *p, *q; char *c; /* amd64 names for some asm constant printouts */ astypnames[INT] = astypnames[UNSIGNED] = "\t.long"; astypnames[LONG] = astypnames[ULONG] = "\t.quad"; gp_offset = addname("gp_offset"); fp_offset = addname("fp_offset"); overflow_arg_area = addname("overflow_arg_area"); reg_save_area = addname("reg_save_area"); rp = bstruct(NULL, STNAME, NULL); p = block(NAME, NIL, NIL, UNSIGNED, 0, 0); soumemb(p, gp_offset, 0); soumemb(p, fp_offset, 0); p->n_type = VOID+PTR; p->n_ap = NULL; soumemb(p, overflow_arg_area, 0); soumemb(p, reg_save_area, 0); nfree(p); q = dclstruct(rp); c = addname("__builtin_va_list"); p = block(LB, bdty(NAME, c), bcon(1), INT, 0, 0); p = tymerge(q, p); p->n_sp = lookup(c, 0); defid(p, TYPEDEF); nfree(q); nfree(p); /* for the static varargs functions */ #define MKN(vn, rn, tp) \ { vn = addname(rn); sp = lookup(vn, SNORMAL); \ sp->sclass = USTATIC; sp->stype = tp; } MKN(gpnext, "__pcc_gpnext", FTN|LONG); MKN(fpnext, "__pcc_fpnext", FTN|DOUBLE); MKN(_1regref, "__pcc_1regref", FTN|VOID|(PTR<<TSHIFT)); MKN(_2regref, "__pcc_2regref", FTN|VOID|(PTR<<TSHIFT)); MKN(memref, "__pcc_memref", FTN|VOID|(PTR<<TSHIFT)); }
/* * read in all of the arena's clump directory, * convert to IEntry format, and bucket sort based * on the first few bits. */ static uint32_t readarenainfo(IEBucks *ib, Arena *arena, uint64_t a, Bloom *b) { IEntry ie; ClumpInfo *ci, *cis; uint32_t clump; int i, n, ok, nskip; if(arena->memstats.clumps) fprint(2, "\tarena %s: %d entries\n", arena->name, arena->memstats.clumps); else fprint(2, "[%s] ", arena->name); cis = MKN(ClumpInfo, ClumpChunks); ok = 0; nskip = 0; memset(&ie, 0, sizeof(IEntry)); for(clump = 0; clump < arena->memstats.clumps; clump += n){ n = ClumpChunks; if(n > arena->memstats.clumps - clump) n = arena->memstats.clumps - clump; if(readclumpinfos(arena, clump, cis, n) != n){ seterr(EOk, "arena directory read failed: %r"); ok = -1; break; } for(i = 0; i < n; i++){ ci = &cis[i]; ie.ia.type = ci->type; ie.ia.size = ci->uncsize; ie.ia.addr = a; a += ci->size + ClumpSize; ie.ia.blocks = (ci->size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog; scorecp(ie.score, ci->score); if(ci->type == VtCorruptType){ if(0) print("! %V %22lld %3d %5d %3d\n", ie.score, ie.ia.addr, ie.ia.type, ie.ia.size, ie.ia.blocks); nskip++; }else sprayientry(ib, &ie); markbloomfilter(b, ie.score); } } free(cis); if(ok < 0) return TWID32; return clump - nskip; }
IEStream* initiestream(Part *part, uint64_t off, uint64_t clumps, uint32_t size) { IEStream *ies; /* out of memory? */ ies = MKZ(IEStream); ies->buf = MKN(uint8_t, size); ies->epos = ies->buf; ies->pos = ies->epos; ies->off = off; ies->n = clumps; ies->size = size; ies->part = part; return ies; }
int findscore(Arena *arena, uchar *score) { IEntry ie; ClumpInfo *ci, *cis; u64int a; u32int clump; int i, n, found; //ZZZ remove fprint? if(arena->memstats.clumps) fprint(2, "reading directory for arena=%s with %d entries\n", arena->name, arena->memstats.clumps); cis = MKN(ClumpInfo, ClumpChunks); found = 0; a = 0; memset(&ie, 0, sizeof(IEntry)); for(clump = 0; clump < arena->memstats.clumps; clump += n){ n = ClumpChunks; if(n > arena->memstats.clumps - clump) n = arena->memstats.clumps - clump; if(readclumpinfos(arena, clump, cis, n) != n){ seterr(EOk, "arena directory read failed: %r"); break; } for(i = 0; i < n; i++){ ci = &cis[i]; if(scorecmp(score, ci->score)==0){ fprint(2, "found at clump=%d with type=%d size=%d csize=%d position=%lld\n", clump + i, ci->type, ci->uncsize, ci->size, a); found++; } a += ci->size + ClumpSize; } } free(cis); return found; }
static int readarenainfo(Arena *arena) { ClumpInfo *ci, *cis; uint32_t clump; int i, n, ok; if(arena->memstats.clumps) fprint(2, "reading directory for arena=%s with %d entries\n", arena->name, arena->memstats.clumps); cis = MKN(ClumpInfo, ClumpChunks); ok = 0; for(clump = 0; clump < arena->memstats.clumps; clump += n){ n = ClumpChunks; if(n > arena->memstats.clumps - clump) n = arena->memstats.clumps - clump; if((i=readclumpinfos(arena, clump, cis, n)) != n){ seterr(EOk, "arena directory read failed %d not %d: %r", i, n); ok = -1; break; } for(i = 0; i < n; i++){ ci = &cis[i]; if(ci->type >= VtMaxType || ci->uncsize >= VtMaxLumpSize) { fprint(2, "bad clump: %d: type = %d: size = %d\n", clump+i, ci->type, ci->uncsize); continue; } count[ci->uncsize][ci->type]++; } } free(cis); if(ok < 0) return TWID32; return clump; }