// // Stolen from fixPolishesIID // void addToDict(dict_t *d, char *n) { dnode_t *node = 0L; char *dcpy = 0L; if (n == 0L) return; seqCache *F = new seqCache(n); seqInCore *S = F->getSequenceInCore(); while (S) { node = (dnode_t *)palloc(sizeof(dnode_t)); dcpy = (char *)palloc(sizeof(char) * S->headerLength() + 1); strcpy(dcpy, S->header()); dnode_init(node, (void *)(unsigned long)S->getIID()); dict_insert(d, node, dcpy); delete S; S = F->getSequenceInCore(); } delete F; }
const void * fc_solve_kaz_tree_alloc_insert(dict_t *dict, const void *key) #endif { dnode_t * from_bin; dnode_t * node; const void * ret; fcs_compact_allocator_t * allocator; allocator = &(dict->dict_allocator); if ((from_bin = dict->dict_recycle_bin) != NULL) { node = dict->dict_recycle_bin; dict->dict_recycle_bin = DNODE_NEXT(node); } else { node = (dnode_t *) fcs_compact_alloc_ptr(allocator, sizeof(*node)) ; } #ifdef NO_FC_SOLVE dnode_init(node, data); #else dnode_init(node); #endif if ((ret = fc_solve_kaz_tree_insert(dict, node, key))) { if (from_bin) { DNODE_NEXT(node) = dict->dict_recycle_bin; dict->dict_recycle_bin = node; } else { fcs_compact_alloc_release(allocator); } } return ret; }
int dict_alloc_insert(dict_t *dict, const void *key, void *data) { dnode_t *node = dict->allocnode(dict->context); if (node) { dnode_init(node, data); dict_insert(dict, node, key); return 1; } return 0; }
int cxml_node_addnode(CLOG_INFO* info, CXMLNODE *parent, CXMLNODE *child) { int return_value=0; assert(NULL!=parent); assert(NULL!=child); // if the sub node is NULL create a new sub table if(NULL==parent->sub) { clog( info, CTRACE, "XML: xml_node_addnode(), creating sub node"); parent->sub = dict_create(DICTCOUNT_T_MAX, (dict_comp_t)strcmp); // allow duplicates dict_allow_dupes(parent->sub); } // if the previous stuff worked, add the name/value pair if(NULL!=parent->sub && NULL!=child->name && NULL!=child->name->string) { dnode_t *node = (parent->sub)->dict_allocnode((parent->sub)->dict_context); char *key = strdup(child->name->string); clog( info, CTRACE, "XML: xml_node_adddata(), Saving sub node name=\"%.80s\"", child->name->string); if (node) { dnode_init(node, child); dict_insert(parent->sub, node, key); // let the child know who they are child->me = node; // success! return_value=1; } } // let the child know whow their parent is if(NULL!=parent && NULL!=child) { child->parent = parent; } return return_value; }
static struct context * core_ctx_create(struct instance *nci) { rstatus_t status; struct context *ctx; srand((unsigned) time(NULL)); ctx = dn_alloc(sizeof(*ctx)); if (ctx == NULL) { return NULL; } ctx->id = ++ctx_id; ctx->cf = NULL; ctx->stats = NULL; ctx->evb = NULL; array_null(&ctx->pool); ctx->max_timeout = nci->stats_interval; ctx->timeout = ctx->max_timeout; ctx->dyn_state = INIT; /* parse and create configuration */ ctx->cf = conf_create(nci->conf_filename); if (ctx->cf == NULL) { loga("Failed to create context!!!"); dn_free(ctx); return NULL; } /* initialize server pool from configuration */ status = server_pool_init(&ctx->pool, &ctx->cf->pool, ctx); if (status != DN_OK) { loga("Failed to initialize server pool!!!"); conf_destroy(ctx->cf); dn_free(ctx); return NULL; } /* crypto init */ status = crypto_init(ctx); if (status != DN_OK) { loga("Failed to initialize crypto!!!"); dn_free(ctx); return NULL; } /* create stats per server pool */ ctx->stats = stats_create(nci->stats_port, nci->stats_addr, nci->stats_interval, nci->hostname, &ctx->pool, ctx); if (ctx->stats == NULL) { loga("Failed to create stats!!!"); crypto_deinit(); server_pool_deinit(&ctx->pool); conf_destroy(ctx->cf); dn_free(ctx); return NULL; } /* initialize event handling for client, proxy and server */ ctx->evb = event_base_create(EVENT_SIZE, &core_core); if (ctx->evb == NULL) { loga("Failed to create socket event handling!!!"); crypto_deinit(); stats_destroy(ctx->stats); server_pool_deinit(&ctx->pool); conf_destroy(ctx->cf); dn_free(ctx); return NULL; } /* preconnect? servers in server pool */ status = server_pool_preconnect(ctx); if (status != DN_OK) { loga("Failed to preconnect for server pool!!!"); crypto_deinit(); server_pool_disconnect(ctx); event_base_destroy(ctx->evb); stats_destroy(ctx->stats); server_pool_deinit(&ctx->pool); conf_destroy(ctx->cf); dn_free(ctx); return NULL; } /* initialize proxy per server pool */ status = proxy_init(ctx); if (status != DN_OK) { loga("Failed to initialize proxy!!!"); crypto_deinit(); server_pool_disconnect(ctx); event_base_destroy(ctx->evb); stats_destroy(ctx->stats); server_pool_deinit(&ctx->pool); conf_destroy(ctx->cf); dn_free(ctx); return NULL; } /* initialize dnode listener per server pool */ status = dnode_init(ctx); if (status != DN_OK) { loga("Failed to initialize dnode!!!"); crypto_deinit(); server_pool_disconnect(ctx); event_base_destroy(ctx->evb); stats_destroy(ctx->stats); server_pool_deinit(&ctx->pool); conf_destroy(ctx->cf); dn_free(ctx); return NULL; } ctx->dyn_state = JOINING; //TODOS: change this to JOINING /* initialize peers */ status = dnode_peer_init(&ctx->pool, ctx); if (status != DN_OK) { loga("Failed to initialize dnode peers!!!"); crypto_deinit(); dnode_deinit(ctx); server_pool_disconnect(ctx); event_base_destroy(ctx->evb); stats_destroy(ctx->stats); server_pool_deinit(&ctx->pool); conf_destroy(ctx->cf); dn_free(ctx); return NULL; } core_debug(ctx); /* preconntect peers - probably start gossip here */ status = dnode_peer_pool_preconnect(ctx); if (status != DN_OK) { loga("Failed to preconnect dnode peers!!!"); crypto_deinit(); dnode_peer_deinit(&ctx->pool); dnode_deinit(ctx); server_pool_disconnect(ctx); event_base_destroy(ctx->evb); stats_destroy(ctx->stats); server_pool_deinit(&ctx->pool); conf_destroy(ctx->cf); dn_free(ctx); return NULL; } //init ring msg queue CBUF_Init(C2G_InQ); CBUF_Init(C2G_OutQ); //init stats msg queue CBUF_Init(C2S_InQ); CBUF_Init(C2S_OutQ); gossip_pool_init(ctx); log_debug(LOG_VVERB, "created ctx %p id %"PRIu32"", ctx, ctx->id); return ctx; }
int main(int argc, char **argv) { char *merylCount = 0L; char *fastaName = 0L; int arg=1; while (arg < argc) { if (strcmp(argv[arg], "-m") == 0) { merylCount = argv[++arg]; } else if (strcmp(argv[arg], "-f") == 0) { fastaName = argv[++arg]; } else { fprintf(stderr, "unknown option '%s'\n", argv[arg]); } arg++; } if ((merylCount == 0L) || (fastaName == 0L)) { fprintf(stderr, "usage: %s -m <meryl-name-prefix> -f <fasta-file>\n", argv[0]); exit(1); } // Open the count files // merylStreamReader *MSR = new merylStreamReader(merylCount); fprintf(stderr, "Mers are "uint32FMT" bases.\n", MSR->merSize()); fprintf(stderr, "There are "uint64FMT" unique (copy = 1) mers.\n", MSR->numberOfUniqueMers()); fprintf(stderr, "There are "uint64FMT" distinct mers.\n", MSR->numberOfDistinctMers()); fprintf(stderr, "There are "uint64FMT" mers total.\n", MSR->numberOfTotalMers()); // Guess how many mers we can fit into 512MB, then report how many chunks we need to do. uint32 merSize = MSR->merSize(); uint64 memoryLimit = 700 * 1024 * 1024; uint64 perMer = sizeof(kMerLite) + sizeof(dnode_t); uint64 mersPerBatch = memoryLimit / perMer; uint32 numBatches = MSR->numberOfDistinctMers() / mersPerBatch; uint32 batch = 0; dnode_t *nodes = new dnode_t [mersPerBatch]; kMerLite *mers = new kMerLite [mersPerBatch]; if (MSR->numberOfDistinctMers() % mersPerBatch) numBatches++; fprintf(stderr, "perMer: "uint64FMT" bytes ("uint64FMT" for kMerLite, "uint64FMT" for dnode_t.\n", perMer, (uint64)sizeof(kMerLite), (uint64)sizeof(dnode_t)); fprintf(stderr, "We can fit "uint64FMT" mers into "uint64FMT"MB.\n", mersPerBatch, memoryLimit >> 20); fprintf(stderr, "So we need "uint32FMT" batches to verify the count.\n", numBatches); while (MSR->validMer()) { uint64 mersRemain = mersPerBatch; dict_t *merDict = dict_create(mersPerBatch, kMerLiteSort); batch++; // STEP 1: Insert mersPerBatch into the merDict // fprintf(stderr, "STEP 1 BATCH "uint32FMTW(2)": Insert into merDict\n", batch); while (MSR->nextMer() && mersRemain) { mersRemain--; mers[mersRemain] = MSR->theFMer(); // initialize the node with the value, then insert the node // into the tree using the key int32 val = (int32)MSR->theCount(); dnode_init(&nodes[mersRemain], (void *)val); dict_insert(merDict, &nodes[mersRemain], &mers[mersRemain]); } // STEP 2: Stream the original file, decrementing the count // fprintf(stderr, "STEP 2 BATCH "uint32FMTW(2)": Stream fasta\n", batch); seqStream *CS = new seqStream(fastaName, true); merStream *MS = new merStream(new kMerBuilder(merSize), CS); kMerLite mer; dnode_t *nod; while (MS->nextMer()) { mer = MS->theFMer(); nod = dict_lookup(merDict, &mer); if (nod != 0L) { int32 val = (int32)dnode_get(nod); val--; dnode_put(nod, (void *)val); } else { // Unless the whole meryl file fit into our merDict, we cannot warn if // we don't find mers. // if (numBatches == 1) { char str[1024]; fprintf(stderr, "Didn't find node for mer '%s'\n", mer.merToString(merSize, str)); } } } delete MS; delete CS; // STEP 3: Check every node in the tree to make sure that the counts // are exactly zero. // fprintf(stderr, "STEP 3 BATCH "uint32FMTW(2)": Check\n", batch); nod = dict_first(merDict); while (nod) { int32 val = (int32)dnode_get(nod); kMerLite const *nodmer = (kMerLite const *)dnode_getkey(nod); if (val != 0) { char str[1024]; fprintf(stderr, "Got count "int32FMT" for mer '%s'\n", val, nodmer->merToString(merSize, str)); } nod = dict_next(merDict, nod); } // STEP 4: Destroy the dictionary. // fprintf(stderr, "STEP 4 BATCH "uint32FMTW(2)": Destroy\n", batch); while ((nod = dict_first(merDict))) dict_delete(merDict, nod); dict_destroy(merDict); } }