void metaDestroy(meta_t *node) { if (node == NULL) return; sdsdel(node->key); if (node->state == IN_CACHE) sstBlockDestroy(node->block); metaFree(node); }
void metaFreeList(struct meta **pList) /* Free a list of dynamically allocated meta's. Use metaFreeForest to free children too. */ { struct meta *el, *next; for (el = *pList; el != NULL; el = next) { next = el->next; metaFree(&el); } *pList = NULL; }
sstindex_t *sstIndexLoadFromSStable(int32_t fd, int32_t offset, int32_t size) { int32_t len; uint64_t magic; meta_t *node; sstindex_t *list; int8_t *buffer, *ptr; if (fd < 0 || offset < 0 || size < 0) return(NULL); if ((list = sstIndexCreate()) == NULL) return(NULL); if ((buffer = malloc(size)) == NULL) { sstIndexFree(list); return(NULL); } do { lseek(fd,offset,SEEK_SET); } while (size != read(fd,buffer,size)); ptr = buffer; magic = *(uint64_t *)ptr; ptr = ptr + sizeof(uint64_t); if (magic != crc32_encode(ptr,size - sizeof(uint64_t))) { sstIndexFree(list); free(buffer); return(NULL); } while (ptr - buffer < size) { if ((node = metaCreate()) == NULL) break; node->offset = *(int32_t *)ptr; ptr = ptr + sizeof(int32_t); node->blocksize = *(int32_t *)ptr; ptr = ptr + sizeof(int32_t); len = *(int32_t *)ptr; ptr = ptr + sizeof(int32_t); node->key = sdsnnew(ptr,len); ptr = ptr + len; if (node->key == NULL) { metaFree(node); continue; } sstIndexInsertMeta(list,node); } free(buffer); return(list); }
struct meta *metaLoadAll(char *fileName, char *keyTag, char *parentTag, boolean ignoreOtherStanzas, boolean ignoreIndent) /* Loads in all ra stanzas from file and turns them into a list of meta, some of which * may have children. The keyTag parameter is optional. If non-null it should be set to * the tag name that starts a stanza. If null, the first tag of the first stanza will be used. * The parentTag if non-NULL will be a tag name used to define the parent of a stanza. * The ignoreOtherStanzas flag if set will ignore stanzas that start with other tags. * If not set the routine will abort on such stanzas. The ignoreIndent if set will * use the parentTag (which must be set) to define the hierarchy. Otherwise the program * will look at the indentation, and if there is a parentTag complain about any * disagreements between indentation and parentTag. */ { struct lineFile *lf = netLineFileOpen(fileName); struct meta *meta, *forest = NULL, *lastMeta = NULL; if (ignoreIndent) { errAbort("Currently metaLoadAll can't ignore indentation, sorry."); } while ((meta = metaNextStanza(lf)) != NULL) { struct meta **pList; if (forest == NULL) /* First time. */ { if (meta->indent != 0) errAbort("Initial stanza of %s should not be indented", fileName); if (keyTag == NULL) keyTag = meta->tagList->tag; pList = &forest; } else { if (!sameString(keyTag, meta->tagList->tag)) { if (ignoreOtherStanzas) { metaFree(&meta); continue; } else errAbort("Stanza beginning with %s instead of %s line %d of %s", meta->tagList->tag, keyTag, lf->lineIx, lf->fileName); } if (meta->indent > lastMeta->indent) { pList = &lastMeta->children; meta->parent = lastMeta; } else if (meta->indent == lastMeta->indent) { if (meta->indent == 0) pList = &forest; else { pList = &lastMeta->parent->children; meta->parent = lastMeta->parent; } } else /* meta->indent < lastMeta->indent */ { /* Find sibling at same level as us. */ struct meta *olderSibling; for (olderSibling = lastMeta->parent; olderSibling != NULL; olderSibling = olderSibling->parent) { if (meta->indent == olderSibling->indent) break; } if (olderSibling == NULL) { warn("Indentation inconsistent in stanza ending line %d of %s.", lf->lineIx, lf->fileName); warn("If you are using tabs, check your tab stop is set to 8."); warn("Otherwise check that when you are reducing indentation in a stanza"); warn("that it is the same as the previous stanza at the same level."); noWarnAbort(); } if (olderSibling->parent == NULL) pList = &forest; else { pList = &olderSibling->parent->children; meta->parent = olderSibling->parent; } } } slAddHead(pList, meta); lastMeta = meta; } lineFileClose(&lf); forest = rReverseMetaList(forest); return forest; }
int32_t ssTableMerge(SST *sst, SST *ssa, SST *ssb) { int32_t sstfd,nsstfd; sstblock_t *bufferBlock; entry_t *aMin, *bMin, *curMin; meta_t *Ameta, *Bmeta, *bufferMeta; if (!ssa || !ssb || !sst) return(-1); if (!sst->metas && (sst->metas = sstIndexCreate()) == NULL) return(-1); if (!sst->trailer && (sst->trailer = sstTrailerCreate()) == NULL) return(-1); if (!sst->fileinfo && (sst->fileinfo = sstInfoCreate()) == NULL) return(-1); if (!sst->bloom && (sst->bloom = sstBloomCreate( ssa->fileinfo->entrycount + ssb->fileinfo->entrycount, BLOOM_P)) == NULL) return(-1); do { nsstfd = open(sst->s_name,O_RDWR|O_CREAT|O_APPEND,0644); } while (nsstfd < 0); /* The Base merge algorithm * * Suppose there are two SStable A and B, being be merged into a new large * SStable C, each time to remove an entry from A and B and save the entries * pointer to n and m, and then take the loop operation: 1. if n == NULL, pick up a min entry from A. as well as B to m. 2. compare key(n) and key(m) : 2.1 if key(n) < key(m) insert n into the new sstable C and then make n = NULL. 2.2 if key(n) > key(m) insert m into the new sstable C and then make m = NULL. 2.3 if key(n) == key(m), compare time(n) and time(m) 2.3.1 if time(n) < time(m) this shows that n was expired. n will be given up and insert m into C. 2.3.2 if time(n) > time(m) It is similar to above, give up m and insert n into C 2.3.3 time(n) == time(m) ? Impossible, the typical logic error. 3. continue loop. */ Ameta = sstIndexHeader(ssa->metas); Bmeta = sstIndexHeader(ssb->metas); aMin = bMin = NULL; while (Ameta || Bmeta) { bufferMeta = metaCreate(); bufferBlock = sstBlockCreate(); if (!bufferMeta || !bufferBlock) { if (bufferMeta) metaFree(bufferMeta); if (bufferBlock) sstBlockFree(bufferBlock); ssTableDestroy(sst); return(-1); } while (sstBlockSize(bufferBlock) < _BLOCK_SIZE) { do { if (!Ameta) break; if (Ameta->state != IN_CACHE) { sstfd = open(ssa->s_name,O_RDONLY); Ameta->block = sstBlockLoadFromSStable( sstfd, Ameta->offset, Ameta->blocksize); Ameta->state = IN_CACHE; close(sstfd); } if (!aMin) { if ((aMin = sstBlockPickupMin(Ameta->block)) == NULL) { Ameta->state = IN_DISK; sstBlockDestroy(Ameta->block); Ameta = metaNext(Ameta); } } } while (!aMin); do { if (!Bmeta) break; if (Bmeta->state != IN_CACHE) { sstfd = open(ssb->s_name,O_RDONLY); Bmeta->block = sstBlockLoadFromSStable( sstfd, Bmeta->offset, Bmeta->blocksize); Bmeta->state = IN_CACHE; close(sstfd); } if (!bMin) { if ((bMin = sstBlockPickupMin(Bmeta->block)) == NULL) { Bmeta->state = IN_DISK; sstBlockDestroy(Bmeta->block); Bmeta = metaNext(Bmeta); } } } while (!bMin); if (!aMin && !bMin) break; else if (aMin && bMin) { if (entryCompare(aMin,bMin) < 0) { curMin = aMin; aMin = NULL; } else if (entryCompare(aMin,bMin) > 0) { curMin = bMin; bMin = NULL; } else { if (ssa->trailer->timestamp > ssb->trailer->timestamp) { curMin = aMin; entryDestroy(bMin); aMin = bMin = NULL; } else { curMin = bMin; entryDestroy(aMin); aMin = bMin = NULL; } } } else { if (!aMin) { curMin = bMin; bMin = NULL; } else { curMin = aMin; aMin = NULL; } } sstBlockInsertEntry(bufferBlock,curMin); } if (bufferBlock->_count > 0) { bufferMeta->key = sdsdup((sstBlockHeader(bufferBlock))->key); bufferMeta->blocksize = sstBlockSize(bufferBlock); sstIndexInsertMeta(sst->metas,bufferMeta); sstBloomInsertKeySets(sst->bloom,bufferBlock); do { bufferMeta->offset = lseek(nsstfd,0,SEEK_END); } while (bufferMeta->blocksize != sstBlockDumpIntoSStable(nsstfd,bufferMeta->offset,bufferBlock)); sst->fileinfo->blockcount++; sst->fileinfo->entrycount = sst->fileinfo->entrycount + bufferBlock->_count; /* free the sstable block cache data */ sstBlockDestroy(bufferBlock); } else { metaDestroy(bufferMeta); sstBlockDestroy(bufferBlock); } } bufferMeta = sstIndexTailer(sst->metas); if (bufferMeta->state != IN_CACHE) bufferMeta->block = sstBlockLoadFromSStable(nsstfd,bufferMeta->offset,bufferMeta->blocksize); sst->fileinfo->lastkey = sdsdup((sstBlockTailer(bufferMeta->block))->key); sstBlockDestroy(bufferMeta->block); bufferMeta->block = NULL; sst->trailer->bloomoffset = lseek(nsstfd,0,SEEK_CUR); sst->trailer->bloomsize = sstBloomDumpIntoSStable(nsstfd,sst->trailer->bloomoffset,sst->bloom); sst->trailer->indexoffset = lseek(nsstfd,0,SEEK_CUR); sst->trailer->indexsize = sstIndexDumpIntoSStable(nsstfd,sst->trailer->indexoffset,sst->metas); sst->trailer->infooffset = lseek(nsstfd,0,SEEK_CUR); sst->trailer->infosize = sstInfoDumpIntoSStable(nsstfd,sst->trailer->infooffset,sst->fileinfo); sstTrailerDumpIntoSStable(nsstfd,lseek(nsstfd,0,SEEK_CUR),sst->trailer); close(nsstfd); close(sstfd); return(0); }