void metaDestroy(meta_t *node)
{
if (node == NULL)
return;
sdsdel(node->key);
if (node->state == IN_CACHE)
sstBlockDestroy(node->block);
metaFree(node);
}
void metaFreeList(struct meta **pList)
/* Free a list of dynamically allocated meta's. Use metaFreeForest to free children too. */
{
struct meta *el, *next;
for (el = *pList; el != NULL; el = next)
{
next = el->next;
metaFree(&el);
}
*pList = NULL;
}
sstindex_t *sstIndexLoadFromSStable(int32_t fd, int32_t offset, int32_t size)
{
int32_t len;
uint64_t magic;
meta_t *node;
sstindex_t *list;
int8_t *buffer, *ptr;
if (fd < 0 || offset < 0 || size < 0)
return(NULL);
if ((list = sstIndexCreate()) == NULL)
return(NULL);
if ((buffer = malloc(size)) == NULL) {
sstIndexFree(list);
return(NULL);
}
do {
lseek(fd,offset,SEEK_SET);
} while (size != read(fd,buffer,size));
ptr = buffer;
magic = *(uint64_t *)ptr;
ptr = ptr + sizeof(uint64_t);
if (magic != crc32_encode(ptr,size - sizeof(uint64_t))) {
sstIndexFree(list);
free(buffer);
return(NULL);
}
while (ptr - buffer < size) {
if ((node = metaCreate()) == NULL)
break;
node->offset = *(int32_t *)ptr;
ptr = ptr + sizeof(int32_t);
node->blocksize = *(int32_t *)ptr;
ptr = ptr + sizeof(int32_t);
len = *(int32_t *)ptr;
ptr = ptr + sizeof(int32_t);
node->key = sdsnnew(ptr,len);
ptr = ptr + len;
if (node->key == NULL) {
metaFree(node);
continue;
}
sstIndexInsertMeta(list,node);
}
free(buffer);
return(list);
}
struct meta *metaLoadAll(char *fileName, char *keyTag, char *parentTag,
boolean ignoreOtherStanzas, boolean ignoreIndent)
/* Loads in all ra stanzas from file and turns them into a list of meta, some of which
* may have children. The keyTag parameter is optional. If non-null it should be set to
* the tag name that starts a stanza. If null, the first tag of the first stanza will be used.
* The parentTag if non-NULL will be a tag name used to define the parent of a stanza.
* The ignoreOtherStanzas flag if set will ignore stanzas that start with other tags.
* If not set the routine will abort on such stanzas. The ignoreIndent if set will
* use the parentTag (which must be set) to define the hierarchy. Otherwise the program
* will look at the indentation, and if there is a parentTag complain about any
* disagreements between indentation and parentTag. */
{
struct lineFile *lf = netLineFileOpen(fileName);
struct meta *meta, *forest = NULL, *lastMeta = NULL;
if (ignoreIndent)
{
errAbort("Currently metaLoadAll can't ignore indentation, sorry.");
}
while ((meta = metaNextStanza(lf)) != NULL)
{
struct meta **pList;
if (forest == NULL) /* First time. */
{
if (meta->indent != 0)
errAbort("Initial stanza of %s should not be indented", fileName);
if (keyTag == NULL)
keyTag = meta->tagList->tag;
pList = &forest;
}
else
{
if (!sameString(keyTag, meta->tagList->tag))
{
if (ignoreOtherStanzas)
{
metaFree(&meta);
continue;
}
else
errAbort("Stanza beginning with %s instead of %s line %d of %s",
meta->tagList->tag, keyTag, lf->lineIx, lf->fileName);
}
if (meta->indent > lastMeta->indent)
{
pList = &lastMeta->children;
meta->parent = lastMeta;
}
else if (meta->indent == lastMeta->indent)
{
if (meta->indent == 0)
pList = &forest;
else
{
pList = &lastMeta->parent->children;
meta->parent = lastMeta->parent;
}
}
else /* meta->indent < lastMeta->indent */
{
/* Find sibling at same level as us. */
struct meta *olderSibling;
for (olderSibling = lastMeta->parent;
olderSibling != NULL; olderSibling = olderSibling->parent)
{
if (meta->indent == olderSibling->indent)
break;
}
if (olderSibling == NULL)
{
warn("Indentation inconsistent in stanza ending line %d of %s.",
lf->lineIx, lf->fileName);
warn("If you are using tabs, check your tab stop is set to 8.");
warn("Otherwise check that when you are reducing indentation in a stanza");
warn("that it is the same as the previous stanza at the same level.");
noWarnAbort();
}
if (olderSibling->parent == NULL)
pList = &forest;
else
{
pList = &olderSibling->parent->children;
meta->parent = olderSibling->parent;
}
}
}
slAddHead(pList, meta);
lastMeta = meta;
}
lineFileClose(&lf);
forest = rReverseMetaList(forest);
return forest;
}
int32_t ssTableMerge(SST *sst, SST *ssa, SST *ssb)
{
int32_t sstfd,nsstfd;
sstblock_t *bufferBlock;
entry_t *aMin, *bMin, *curMin;
meta_t *Ameta, *Bmeta, *bufferMeta;
if (!ssa || !ssb || !sst)
return(-1);
if (!sst->metas && (sst->metas = sstIndexCreate()) == NULL)
return(-1);
if (!sst->trailer && (sst->trailer = sstTrailerCreate()) == NULL)
return(-1);
if (!sst->fileinfo && (sst->fileinfo = sstInfoCreate()) == NULL)
return(-1);
if (!sst->bloom && (sst->bloom = sstBloomCreate(
ssa->fileinfo->entrycount + ssb->fileinfo->entrycount, BLOOM_P)) == NULL)
return(-1);
do {
nsstfd = open(sst->s_name,O_RDWR|O_CREAT|O_APPEND,0644);
} while (nsstfd < 0);
/* The Base merge algorithm
*
* Suppose there are two SStable A and B, being be merged into a new large
* SStable C, each time to remove an entry from A and B and save the entries
* pointer to n and m, and then take the loop operation:
1. if n == NULL, pick up a min entry from A. as well as B to m.
2. compare key(n) and key(m) :
2.1 if key(n) < key(m)
insert n into the new sstable C and then make n = NULL.
2.2 if key(n) > key(m)
insert m into the new sstable C and then make m = NULL.
2.3 if key(n) == key(m), compare time(n) and time(m)
2.3.1 if time(n) < time(m)
this shows that n was expired. n will be given up and
insert m into C.
2.3.2 if time(n) > time(m)
It is similar to above, give up m and insert n into C
2.3.3 time(n) == time(m) ?
Impossible, the typical logic error.
3. continue loop.
*/
Ameta = sstIndexHeader(ssa->metas);
Bmeta = sstIndexHeader(ssb->metas);
aMin = bMin = NULL;
while (Ameta || Bmeta) {
bufferMeta = metaCreate();
bufferBlock = sstBlockCreate();
if (!bufferMeta || !bufferBlock) {
if (bufferMeta)
metaFree(bufferMeta);
if (bufferBlock)
sstBlockFree(bufferBlock);
ssTableDestroy(sst);
return(-1);
}
while (sstBlockSize(bufferBlock) < _BLOCK_SIZE) {
do {
if (!Ameta)
break;
if (Ameta->state != IN_CACHE) {
sstfd = open(ssa->s_name,O_RDONLY);
Ameta->block = sstBlockLoadFromSStable(
sstfd, Ameta->offset, Ameta->blocksize);
Ameta->state = IN_CACHE;
close(sstfd);
}
if (!aMin) {
if ((aMin = sstBlockPickupMin(Ameta->block)) == NULL) {
Ameta->state = IN_DISK;
sstBlockDestroy(Ameta->block);
Ameta = metaNext(Ameta);
}
}
} while (!aMin);
do {
if (!Bmeta)
break;
if (Bmeta->state != IN_CACHE) {
sstfd = open(ssb->s_name,O_RDONLY);
Bmeta->block = sstBlockLoadFromSStable(
sstfd, Bmeta->offset, Bmeta->blocksize);
Bmeta->state = IN_CACHE;
close(sstfd);
}
if (!bMin) {
if ((bMin = sstBlockPickupMin(Bmeta->block)) == NULL) {
Bmeta->state = IN_DISK;
sstBlockDestroy(Bmeta->block);
Bmeta = metaNext(Bmeta);
}
}
} while (!bMin);
if (!aMin && !bMin)
break;
else if (aMin && bMin) {
if (entryCompare(aMin,bMin) < 0) {
curMin = aMin;
aMin = NULL;
} else if (entryCompare(aMin,bMin) > 0) {
curMin = bMin;
bMin = NULL;
} else {
if (ssa->trailer->timestamp > ssb->trailer->timestamp) {
curMin = aMin;
entryDestroy(bMin);
aMin = bMin = NULL;
} else {
curMin = bMin;
entryDestroy(aMin);
aMin = bMin = NULL;
}
}
} else {
if (!aMin) {
curMin = bMin;
bMin = NULL;
} else {
curMin = aMin;
aMin = NULL;
}
}
sstBlockInsertEntry(bufferBlock,curMin);
}
if (bufferBlock->_count > 0) {
bufferMeta->key = sdsdup((sstBlockHeader(bufferBlock))->key);
bufferMeta->blocksize = sstBlockSize(bufferBlock);
sstIndexInsertMeta(sst->metas,bufferMeta);
sstBloomInsertKeySets(sst->bloom,bufferBlock);
do {
bufferMeta->offset = lseek(nsstfd,0,SEEK_END);
} while (bufferMeta->blocksize !=
sstBlockDumpIntoSStable(nsstfd,bufferMeta->offset,bufferBlock));
sst->fileinfo->blockcount++;
sst->fileinfo->entrycount = sst->fileinfo->entrycount + bufferBlock->_count;
/* free the sstable block cache data */
sstBlockDestroy(bufferBlock);
} else {
metaDestroy(bufferMeta);
sstBlockDestroy(bufferBlock);
}
}
bufferMeta = sstIndexTailer(sst->metas);
if (bufferMeta->state != IN_CACHE)
bufferMeta->block = sstBlockLoadFromSStable(nsstfd,bufferMeta->offset,bufferMeta->blocksize);
sst->fileinfo->lastkey = sdsdup((sstBlockTailer(bufferMeta->block))->key);
sstBlockDestroy(bufferMeta->block);
bufferMeta->block = NULL;
sst->trailer->bloomoffset = lseek(nsstfd,0,SEEK_CUR);
sst->trailer->bloomsize = sstBloomDumpIntoSStable(nsstfd,sst->trailer->bloomoffset,sst->bloom);
sst->trailer->indexoffset = lseek(nsstfd,0,SEEK_CUR);
sst->trailer->indexsize = sstIndexDumpIntoSStable(nsstfd,sst->trailer->indexoffset,sst->metas);
sst->trailer->infooffset = lseek(nsstfd,0,SEEK_CUR);
sst->trailer->infosize = sstInfoDumpIntoSStable(nsstfd,sst->trailer->infooffset,sst->fileinfo);
sstTrailerDumpIntoSStable(nsstfd,lseek(nsstfd,0,SEEK_CUR),sst->trailer);
close(nsstfd);
close(sstfd);
return(0);
}
