//
// Stolen from fixPolishesIID
//
void
addToDict(dict_t *d, char *n) {
dnode_t *node = 0L;
char *dcpy = 0L;
if (n == 0L)
return;
seqCache *F = new seqCache(n);
seqInCore *S = F->getSequenceInCore();
while (S) {
node = (dnode_t *)palloc(sizeof(dnode_t));
dcpy = (char *)palloc(sizeof(char) * S->headerLength() + 1);
strcpy(dcpy, S->header());
dnode_init(node, (void *)(unsigned long)S->getIID());
dict_insert(d, node, dcpy);
delete S;
S = F->getSequenceInCore();
}
delete F;
}
const void * fc_solve_kaz_tree_alloc_insert(dict_t *dict, const void *key)
#endif
{
dnode_t * from_bin;
dnode_t * node;
const void * ret;
fcs_compact_allocator_t * allocator;
allocator = &(dict->dict_allocator);
if ((from_bin = dict->dict_recycle_bin) != NULL)
{
node = dict->dict_recycle_bin;
dict->dict_recycle_bin = DNODE_NEXT(node);
}
else
{
node = (dnode_t *)
fcs_compact_alloc_ptr(allocator, sizeof(*node))
;
}
#ifdef NO_FC_SOLVE
dnode_init(node, data);
#else
dnode_init(node);
#endif
if ((ret = fc_solve_kaz_tree_insert(dict, node, key)))
{
if (from_bin)
{
DNODE_NEXT(node) = dict->dict_recycle_bin;
dict->dict_recycle_bin = node;
}
else
{
fcs_compact_alloc_release(allocator);
}
}
return ret;
}
int dict_alloc_insert(dict_t *dict, const void *key, void *data)
{
dnode_t *node = dict->allocnode(dict->context);
if (node) {
dnode_init(node, data);
dict_insert(dict, node, key);
return 1;
}
return 0;
}
int cxml_node_addnode(CLOG_INFO* info, CXMLNODE *parent, CXMLNODE *child)
{
int return_value=0;
assert(NULL!=parent);
assert(NULL!=child);
// if the sub node is NULL create a new sub table
if(NULL==parent->sub) {
clog( info, CTRACE, "XML: xml_node_addnode(), creating sub node");
parent->sub = dict_create(DICTCOUNT_T_MAX, (dict_comp_t)strcmp);
// allow duplicates
dict_allow_dupes(parent->sub);
}
// if the previous stuff worked, add the name/value pair
if(NULL!=parent->sub && NULL!=child->name && NULL!=child->name->string) {
dnode_t *node = (parent->sub)->dict_allocnode((parent->sub)->dict_context);
char *key = strdup(child->name->string);
clog( info, CTRACE, "XML: xml_node_adddata(), Saving sub node name=\"%.80s\"", child->name->string);
if (node) {
dnode_init(node, child);
dict_insert(parent->sub, node, key);
// let the child know who they are
child->me = node;
// success!
return_value=1;
}
}
// let the child know whow their parent is
if(NULL!=parent && NULL!=child) {
child->parent = parent;
}
return return_value;
}
static struct context *
core_ctx_create(struct instance *nci)
{
rstatus_t status;
struct context *ctx;
srand((unsigned) time(NULL));
ctx = dn_alloc(sizeof(*ctx));
if (ctx == NULL) {
return NULL;
}
ctx->id = ++ctx_id;
ctx->cf = NULL;
ctx->stats = NULL;
ctx->evb = NULL;
array_null(&ctx->pool);
ctx->max_timeout = nci->stats_interval;
ctx->timeout = ctx->max_timeout;
ctx->dyn_state = INIT;
/* parse and create configuration */
ctx->cf = conf_create(nci->conf_filename);
if (ctx->cf == NULL) {
loga("Failed to create context!!!");
dn_free(ctx);
return NULL;
}
/* initialize server pool from configuration */
status = server_pool_init(&ctx->pool, &ctx->cf->pool, ctx);
if (status != DN_OK) {
loga("Failed to initialize server pool!!!");
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
/* crypto init */
status = crypto_init(ctx);
if (status != DN_OK) {
loga("Failed to initialize crypto!!!");
dn_free(ctx);
return NULL;
}
/* create stats per server pool */
ctx->stats = stats_create(nci->stats_port, nci->stats_addr, nci->stats_interval,
nci->hostname, &ctx->pool, ctx);
if (ctx->stats == NULL) {
loga("Failed to create stats!!!");
crypto_deinit();
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
/* initialize event handling for client, proxy and server */
ctx->evb = event_base_create(EVENT_SIZE, &core_core);
if (ctx->evb == NULL) {
loga("Failed to create socket event handling!!!");
crypto_deinit();
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
/* preconnect? servers in server pool */
status = server_pool_preconnect(ctx);
if (status != DN_OK) {
loga("Failed to preconnect for server pool!!!");
crypto_deinit();
server_pool_disconnect(ctx);
event_base_destroy(ctx->evb);
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
/* initialize proxy per server pool */
status = proxy_init(ctx);
if (status != DN_OK) {
loga("Failed to initialize proxy!!!");
crypto_deinit();
server_pool_disconnect(ctx);
event_base_destroy(ctx->evb);
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
/* initialize dnode listener per server pool */
status = dnode_init(ctx);
if (status != DN_OK) {
loga("Failed to initialize dnode!!!");
crypto_deinit();
server_pool_disconnect(ctx);
event_base_destroy(ctx->evb);
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
ctx->dyn_state = JOINING; //TODOS: change this to JOINING
/* initialize peers */
status = dnode_peer_init(&ctx->pool, ctx);
if (status != DN_OK) {
loga("Failed to initialize dnode peers!!!");
crypto_deinit();
dnode_deinit(ctx);
server_pool_disconnect(ctx);
event_base_destroy(ctx->evb);
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
core_debug(ctx);
/* preconntect peers - probably start gossip here */
status = dnode_peer_pool_preconnect(ctx);
if (status != DN_OK) {
loga("Failed to preconnect dnode peers!!!");
crypto_deinit();
dnode_peer_deinit(&ctx->pool);
dnode_deinit(ctx);
server_pool_disconnect(ctx);
event_base_destroy(ctx->evb);
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
//init ring msg queue
CBUF_Init(C2G_InQ);
CBUF_Init(C2G_OutQ);
//init stats msg queue
CBUF_Init(C2S_InQ);
CBUF_Init(C2S_OutQ);
gossip_pool_init(ctx);
log_debug(LOG_VVERB, "created ctx %p id %"PRIu32"", ctx, ctx->id);
return ctx;
}
int
main(int argc, char **argv) {
char *merylCount = 0L;
char *fastaName = 0L;
int arg=1;
while (arg < argc) {
if (strcmp(argv[arg], "-m") == 0) {
merylCount = argv[++arg];
} else if (strcmp(argv[arg], "-f") == 0) {
fastaName = argv[++arg];
} else {
fprintf(stderr, "unknown option '%s'\n", argv[arg]);
}
arg++;
}
if ((merylCount == 0L) || (fastaName == 0L)) {
fprintf(stderr, "usage: %s -m <meryl-name-prefix> -f <fasta-file>\n", argv[0]);
exit(1);
}
// Open the count files
//
merylStreamReader *MSR = new merylStreamReader(merylCount);
fprintf(stderr, "Mers are "uint32FMT" bases.\n", MSR->merSize());
fprintf(stderr, "There are "uint64FMT" unique (copy = 1) mers.\n", MSR->numberOfUniqueMers());
fprintf(stderr, "There are "uint64FMT" distinct mers.\n", MSR->numberOfDistinctMers());
fprintf(stderr, "There are "uint64FMT" mers total.\n", MSR->numberOfTotalMers());
// Guess how many mers we can fit into 512MB, then report how many chunks we need to do.
uint32 merSize = MSR->merSize();
uint64 memoryLimit = 700 * 1024 * 1024;
uint64 perMer = sizeof(kMerLite) + sizeof(dnode_t);
uint64 mersPerBatch = memoryLimit / perMer;
uint32 numBatches = MSR->numberOfDistinctMers() / mersPerBatch;
uint32 batch = 0;
dnode_t *nodes = new dnode_t [mersPerBatch];
kMerLite *mers = new kMerLite [mersPerBatch];
if (MSR->numberOfDistinctMers() % mersPerBatch)
numBatches++;
fprintf(stderr, "perMer: "uint64FMT" bytes ("uint64FMT" for kMerLite, "uint64FMT" for dnode_t.\n",
perMer, (uint64)sizeof(kMerLite), (uint64)sizeof(dnode_t));
fprintf(stderr, "We can fit "uint64FMT" mers into "uint64FMT"MB.\n", mersPerBatch, memoryLimit >> 20);
fprintf(stderr, "So we need "uint32FMT" batches to verify the count.\n", numBatches);
while (MSR->validMer()) {
uint64 mersRemain = mersPerBatch;
dict_t *merDict = dict_create(mersPerBatch, kMerLiteSort);
batch++;
// STEP 1: Insert mersPerBatch into the merDict
//
fprintf(stderr, "STEP 1 BATCH "uint32FMTW(2)": Insert into merDict\n", batch);
while (MSR->nextMer() && mersRemain) {
mersRemain--;
mers[mersRemain] = MSR->theFMer();
// initialize the node with the value, then insert the node
// into the tree using the key
int32 val = (int32)MSR->theCount();
dnode_init(&nodes[mersRemain], (void *)val);
dict_insert(merDict, &nodes[mersRemain], &mers[mersRemain]);
}
// STEP 2: Stream the original file, decrementing the count
//
fprintf(stderr, "STEP 2 BATCH "uint32FMTW(2)": Stream fasta\n", batch);
seqStream *CS = new seqStream(fastaName, true);
merStream *MS = new merStream(new kMerBuilder(merSize), CS);
kMerLite mer;
dnode_t *nod;
while (MS->nextMer()) {
mer = MS->theFMer();
nod = dict_lookup(merDict, &mer);
if (nod != 0L) {
int32 val = (int32)dnode_get(nod);
val--;
dnode_put(nod, (void *)val);
} else {
// Unless the whole meryl file fit into our merDict, we cannot warn if
// we don't find mers.
//
if (numBatches == 1) {
char str[1024];
fprintf(stderr, "Didn't find node for mer '%s'\n", mer.merToString(merSize, str));
}
}
}
delete MS;
delete CS;
// STEP 3: Check every node in the tree to make sure that the counts
// are exactly zero.
//
fprintf(stderr, "STEP 3 BATCH "uint32FMTW(2)": Check\n", batch);
nod = dict_first(merDict);
while (nod) {
int32 val = (int32)dnode_get(nod);
kMerLite const *nodmer = (kMerLite const *)dnode_getkey(nod);
if (val != 0) {
char str[1024];
fprintf(stderr, "Got count "int32FMT" for mer '%s'\n",
val,
nodmer->merToString(merSize, str));
}
nod = dict_next(merDict, nod);
}
// STEP 4: Destroy the dictionary.
//
fprintf(stderr, "STEP 4 BATCH "uint32FMTW(2)": Destroy\n", batch);
while ((nod = dict_first(merDict)))
dict_delete(merDict, nod);
dict_destroy(merDict);
}
}
