/// Clear out the raw set and cooked sets. Destroy the entire
/// cooked data structure but leave the raw one present though
/// empty. This is the same as the initial (post-creation) state.
/// @param[in] ca the object pointer
void
ca_clear_pa(ca_o ca)
{
dict_t *dict;
dnode_t *dnp, *next;
pa_o pa;
if ((dict = ca->ca_raw_pa_dict)) {
for (dnp = dict_first(dict); dnp;) {
next = dict_next(dict, dnp);
pa = (pa_o)dnode_getkey(dnp);
dict_delete(dict, dnp);
dnode_destroy(dnp);
pa_destroy(pa);
dnp = next;
}
}
if ((dict = ca->ca_cooked_pa_dict)) {
for (dnp = dict_first(dict); dnp;) {
next = dict_next(dict, dnp);
pa = (pa_o)dnode_getkey(dnp);
dict_delete(dict, dnp);
dnode_destroy(dnp);
pa_destroy(pa);
dnp = next;
}
dict_destroy(ca->ca_cooked_pa_dict);
ca->ca_cooked_pa_dict = NULL;
}
}
int main(){
struct dictionary *d;
int i;
d = dict_new(1,0);
int min;
int nums[7];
int keys[7];
keys[0] = 9;
nums[0] = 14;
keys[1] = 4;
nums[1] = 7;
keys[2] = 12;
nums[2] = 5;
keys[3] = 0;
nums[3] = 15;
keys[4] = 2;
nums[4] = 3;
keys[5] = 14;
nums[5] = 4;
keys[6] = 5;
nums[6] = 2;
printf("new\n");
//~ printHash(d);
for(i = 0; i < 7; i++){
//~ printf("Inserting %d...\n",nums[i]);
dict_set(d,keys[i],nums[i]);
//~ printHash(d);
}
printHash(d);
dict_set(d,2,8);
printf("set key 2 con 8\n");
printf("get key 2 = %d\n",dict_get(d,2));
printf("delete key 2\n");
dict_delete(d,2);
printHash(d);
printf("delete key 9\n");
dict_delete(d,9);
printHash(d);
dict_free(d);
}
/*
* #undef
* (C99 6.10.5 Scope of macro definisions, paragraph 2)
*/
static void read_undef(CppContext *ctx) {
Token *name = read_cpp_token(ctx);
if (!name || name->toktype != TOKTYPE_IDENT)
error_token(name, "undef works only to an identifier, but got '%s'", token_to_string(name));
expect_newline(ctx);
dict_delete(ctx->defs, name->val.str);
}
void dict_group_delete(DictGroup* dict_group) {
size_t i;
for (i = 0; i < dict_group->count; i++) {
dict_delete(dict_group->dicts[i]);
}
free(dict_group);
}
void
qserver_cleanup(UNUSED_ARG(void *extra))
{
unsigned int ii;
ioset_close(qserver_listener, 1);
for (ii = 0; ii < qserver_nbots; ++ii)
if (qserver_clients[ii])
DelUser(qserver_clients[ii]->user, NULL, 0, "module finalizing");
dict_delete(qserver_dict);
}
void ui_finish() {
for(int i = 0; i < num_ui_events; i++) {
free(ui_events[i].name);
}
for(int i = 0; i < ui_elem_names->num_items; i++) {
char* name = list_get(ui_elem_names, i);
int* type_id = dict_get(ui_elem_types, name);
debug("Deleting UI Element %s (%s)", name, type_id_name(*type_id));
ui_elem_delete(name);
}
list_delete_with(ui_elem_names, free);
dict_delete(ui_elems);
dict_map(ui_elem_types, free);
dict_delete(ui_elem_types);
}
void
free_record_data(void *rdata)
{
struct record_data *r = rdata;
switch (r->type) {
case RECDB_INVALID: break;
case RECDB_QSTRING: free(r->d.qstring); break;
case RECDB_OBJECT: dict_delete(r->d.object); break;
case RECDB_STRING_LIST: free_string_list(r->d.slist); break;
}
free(r);
}
/**
* Resizes a dict object.
*
* @param struct dict *dict
* @param uint32_t capacity
* @return int
*
* Returns 1 on success, and 0 on error.
**/
int dict_resize(struct dict *dict, uint32_t capacity)
{
struct dict *dict_tmp;
struct dict_node *cur;
struct dict_node *table_tmp;
uint32_t capacity_tmp;
size_t i;
// We pass in NULL as the free functions, otherwise we'll destroy memory
// that we want to keep around.
dict_tmp = dict_new(dict->seed, capacity, NULL, NULL);
if (dict_tmp == NULL) {
return 0;
}
for (i = 0; i < dict->capacity; i++) {
cur = &dict->table[i];
while (cur && cur->key != NULL) {
if (!dict_set(dict_tmp, cur->key, cur->value)) {
dict_delete(dict_tmp);
return 0;
}
cur = cur->next;
}
}
table_tmp = dict->table;
capacity_tmp = dict->capacity;
dict->table = dict_tmp->table;
dict->capacity = dict_tmp->capacity;
dict_tmp->table = table_tmp;
dict_tmp->capacity = capacity_tmp;
dict_delete(dict_tmp);
return 1;
}
static void on_timer(nw_timer *t, void *privdata)
{
double now = current_timestamp();
dict_iterator *iter = dict_get_iterator(dict_update);
dict_entry *entry;
while ((entry = dict_next(iter)) != NULL) {
struct update_val *val = entry->val;
if (val->create_time < (now - 86400)) {
dict_delete(dict_update, entry->key);
}
}
dict_release_iterator(iter);
}
static void
hash_cleanup(UNUSED_ARG(void *extra))
{
dict_iterator_t it, next;
DelServer(self, 0, NULL);
for (it = dict_first(channels); it; it = next) {
next = iter_next(it);
DelChannel(iter_data(it));
}
dict_delete(channels);
dict_delete(clients);
dict_delete(servers);
userList_clean(&curr_opers);
count_opers = 0;
free(slf_list);
free(slf_list_extra);
free(nuf_list);
free(nuf_list_extra);
free(ncf2_list);
free(ncf2_list_extra);
free(duf_list);
free(duf_list_extra);
free(ncf_list);
free(ncf_list_extra);
free(jf_list);
free(jf_list_extra);
free(dcf_list);
free(dcf_list_extra);
free(pf_list);
free(pf_list_extra);
free(kf_list);
free(kf_list_extra);
free(tf_list);
free(tf_list_extra);
}
/// Serializes the CmdAction and writes it to the specified file descriptor.
/// On Windows this must send to a socket. On Unix it can use any
/// kind of file descriptor.
/// @param[in] ca the object pointer
/// @param[in] fd file descriptor to which the serialized form is sent
void
ca_write(ca_o ca, int fd)
{
dict_t *dict;
dnode_t *dnp, *next;
pa_o pa;
CCS pabuf;
dict = ca->ca_raw_pa_dict;
if (dict_count(dict) <= 0) {
return;
}
// In case buffered files were left open by sloppy audited programs.
fflush(NULL);
for (dnp = dict_first(dict); dnp;) {
pa = (pa_o)dnode_getkey(dnp);
// Unfortunately it's too early to dcode any written files
// since they may still be open. But we should get stats of
// non-member read ops since they may differ in the event
// of a distributed build. We must assume logical coherence
// for all write ops, member and non-member.
// In other words, read ops are allowed to be on physically
// separate files with the same path (e.g. /usr/include/stdio.h)
// but write ops to the same path are assumed to be to a shared file.
// This matters in the case of a distributed build.
if (pa_is_read(pa) && !pa_is_member(pa)) {
(void)pa_stat(pa, 0);
}
if ((pabuf = pa_toCSVString(pa))) {
if (write(fd, pabuf, strlen(pabuf)) == -1) {
putil_syserr(0, "write()");
}
putil_free(pabuf);
}
// Dictionary bookkeeping.
next = dict_next(dict, dnp);
dict_delete(dict, dnp);
dnp = next;
pa_destroy(pa);
}
}
/**
* Clone dict object, using key_clone_fn and value_clone_fn, function pointers.
*
* If the clone function pointer for key/value is NULL, then it will simply do a
* static copy of the pointer. Doing this on heap allocated memory may lead to
* memory errors.
*
* @param struct dict *to_clone
* @param void *(*key_clone_fn)(void *)
* @param void *(*value_clone_fn)(void *)
* @return struct dict *
*
* Returns NULL on error.
**/
struct dict *dict_clone(struct dict *to_clone, void *(*key_clone_fn)(void *), void *(*value_clone_fn)(void *))
{
size_t i;
void *key_clone;
void *value_clone;
struct dict_node *cur;
struct dict *clone = dict_new(
to_clone->seed,
to_clone->capacity,
to_clone->key_free_fn,
to_clone->value_free_fn
);
if (clone == NULL) {
return NULL;
}
if (key_clone_fn == NULL) {
key_clone_fn = _dummy_clone_fn;
}
if (value_clone_fn == NULL) {
value_clone_fn = _dummy_clone_fn;
}
for (i = 0; i < to_clone->capacity; i++) {
cur = &to_clone->table[i];
while (cur && cur->key) {
key_clone = key_clone_fn(cur->key);
value_clone = value_clone_fn(cur->value);
if (dict_set(clone, key_clone, value_clone) == 0) {
// Make sure key/value gets freed
clone->key_free_fn(key_clone);
clone->value_free_fn(value_clone);
dict_delete(clone);
return NULL;
}
cur = cur->next;
}
}
return clone;
}
/// Merge two CmdActions, a 'leader' and a 'donor', together. The donor
/// is then redundant.
/// @param[in] leader the leader object pointer
/// @param[in] donor the donor object pointer
void
ca_merge(ca_o leader, ca_o donor)
{
CCS sub;
dict_t *dict;
dnode_t *dnp, *next;
pa_o pa;
// ASSUMPTION: aggregated CAs are composed from
// commands run serially - i.e. any parallelism
// is above the build-script level.
// Stringify the donor and leave that string representation in
// the leader for the record.
sub = ca_format_header(donor);
if (leader->ca_subs) {
leader->ca_subs = (CCS)putil_realloc((void *)leader->ca_subs,
(strlen(leader->ca_subs) +
strlen(sub) +
1) * CHARSIZE);
strcat((CS)leader->ca_subs, sub);
} else {
leader->ca_subs = (CCS)putil_malloc((strlen(sub) + 1) * CHARSIZE);
strcpy((CS)leader->ca_subs, sub);
}
putil_free(sub);
// Traverse the donor's raw list of PA's and move them into the leader's.
if ((dict = donor->ca_raw_pa_dict)) {
for (dnp = dict_first(dict); dnp;) {
next = dict_next(dict, dnp);
pa = (pa_o)dnode_getkey(dnp);
ca_record_pa(leader, pa);
dict_delete(dict, dnp);
dnode_destroy(dnp);
dnp = next;
}
dict_destroy(donor->ca_raw_pa_dict);
donor->ca_raw_pa_dict = NULL;
}
}
TEST(mydict, dict)
{
dict_t* d = dict_create((void*)123);
int k, v;
dict_node_t* node = NULL;
int i = 0;
int sum = 0;
for (i = 0; i < 10000; i++) {
k = i, v = i;
node = dict_add(d, &k, sizeof(k), &v, sizeof(v));
ASSERT_TRUE(node != NULL);
sum += i;
}
for (i = 0; i < 10000; i++) {
node = dict_find(d, &k, sizeof(k));
ASSERT_EQ((long)dict_get_val(node), i);
}
i = 0;
for(node = dict_first(d); node; node = dict_next(d, node)) {
i += (long)dict_get_val(node);
}
ASSERT_EQ(i, sum);
for (i = 0; i < 10000; i++) {
k = i;
node = dict_delete(d, &k, sizeof(k));
ASSERT_EQ((long)dict_get_val(node), i);
}
ASSERT_TRUE(dict_empty(d));
dict_destroy(d);
}
int64_t decompress(const char* in_filename, const char* out_filename, uint8_t flags) {
struct bitio *bd = bstdin;
struct dictionary *d = NULL;
struct utimbuf *t = NULL;
FILE* fout = stdout;
char *out_file = NULL;
uint8_t bits, initial_bits, meta_type, meta_size;
uint16_t c;
uint32_t bitMask, cur, first_record, len, next_record, dict_size = 0, written = 0, write_count = 0;
uint64_t filesize = 0;
char *word;
int first = 1, md5c_size = 0, md5d_size = 0;
void *meta_data, *md5c = NULL, *md5d = NULL;
EVP_MD_CTX *md_ctx = NULL;
if (in_filename != NULL) {
bd = bitio_open(in_filename, 'r');
if (bd == NULL)
goto error;
}
//read metadata
while ((meta_data = meta_read(bd, &meta_type, &meta_size)) != META_END) {
LOG("META_TYPE: %d", meta_type);
switch (meta_type) {
case META_DICT_SIZE:
dict_size = *(uint32_t*)meta_data;
PRINT(1, "Dictionary Size:\t%d\n", dict_size);
break;
case META_NAME:
PRINT(1, "Original file name:\t%s\n", (char*)meta_data);
if (flags & DEC_ORIG_FILENAME) {
out_file = malloc(meta_size);
if (out_file == NULL)
goto error;
memcpy((void*)out_file, meta_data, meta_size);
out_filename = out_file;
}
break;
case META_MD5:
md5c = malloc(meta_size);
memcpy(md5c, meta_data, meta_size);
md5c_size = meta_size;
word = sprinth(md5c, md5c_size);
PRINT(1, "Original md5sum:\t%s\n", word);
free(word);
// initialize md context
OpenSSL_add_all_digests();
md_ctx = malloc(sizeof(EVP_MD_CTX));
EVP_MD_CTX_init(md_ctx);
EVP_DigestInit(md_ctx, EVP_get_digestbyname("md5"));
md5d_size = EVP_MD_CTX_size(md_ctx);
md5d = malloc(md5d_size);
break;
case META_TIMESTAMP:
t = malloc(sizeof(*t));
t->actime = *((time_t*)meta_data); // access time
t->modtime = *((time_t*)meta_data); // modification time
break;
default: // META_ERROR
LOG("Unknown metadata");
errno = EINVAL;
goto error;
}
free(meta_data);
}
if ((flags & DEC_ORIG_FILENAME) && out_file == NULL) // if i have DEC_ORIG_FILENAME setted but no info in metadata i use stdin as outfile
out_filename = "stdin";
if (out_filename != NULL) {
fout = fopen(out_filename, "w");
if (fout == NULL)
goto error;
}
if (out_filename != NULL && in_filename != NULL && strcmp(in_filename, out_filename) == 0) {
errno = EINVAL;
goto error;
}
if (dict_size == 0)
goto error;
d = dict_new(dict_size, 0, dict_size, NUM_SYMBOLS);
if (d == NULL)
goto error;
first_record = dict_init(d);
next_record = first_record;
initial_bits = 0;
bitMask = 1;
while (bitMask < next_record) {
bitMask <<= 1;
initial_bits++;
}
bits = initial_bits;
for (;;) {
// put in cur the index of the fetched word in the dictionary
cur = fetch(bd, bits);
if (cur == ROOT_NODE)
goto error;
if (cur == EOF_SYMBOL)
break;
c = dict_first_symbol(d, cur);
if (c == EOF_SYMBOL)
goto error;
if (!first) {
// complete previous record with index of new record
// ROOT_NODE as current node value means 'don't change it'.
dict_fill(d, next_record, ROOT_NODE, (uint8_t) c, 0);
next_record++;
if ((next_record+1) & bitMask) {
bitMask <<= 1;
bits++;
}
}
else
first = 0;
// get the word in the dictionary at index cur.
word = dict_word(d, cur, &len);
if (word == NULL)
goto error;
written = fwrite(word, 1, len, fout);
if (written < len)
goto error;
else { // md5 computation and visual feedback
if (md5c != NULL) // compute md5 of decompressed
EVP_DigestUpdate(md_ctx, word, len);
write_count += written;
if (write_count >= COUNT_THRESHOLD) {
filesize += write_count;
write_count = 0;
PRINT(1, ".");
}
}
if (next_record + 1 == dict_size) {
next_record = first_record;
bits = initial_bits;
bitMask = 1 << bits;
first = 1; // set first iteration to be the next
}
// add a new record
dict_fill(d, next_record, cur, 0, 0); // symbol will be filled at the beginning of next iteration
}
filesize += write_count;
if (md5c != NULL) {
EVP_DigestFinal_ex(md_ctx, md5d, (unsigned int*)&md5d_size);
if (md5c_size == md5d_size && memcmp(md5c, md5d, md5c_size) == 0)
PRINT(1, "\nmd5sum Check:\t\tOK");
else {
PRINT(1, "\nmd5sum Check:\t\tFailed");
goto error;
}
}
PRINT(1, "\nDecompression Finished\n\n");
fclose(fout);
if (out_file != NULL && t != NULL)
if (utime(out_filename, t) < 0) { // set modification time
PRINT(1, "Error while changing last modification time");
}
free(out_file);
free(t);
dict_delete(d);
bitio_flush(bd);
if (bd != bstdin)
bitio_close(bd);
return filesize;
error:
PRINT(1, "\n");
if (out_filename != NULL)
unlink(out_filename);
free(out_file);
free(t);
dict_delete(d);
bitio_flush(bd);
if (bd != bstdin)
bitio_close(bd);
if (fout != NULL)
fclose(fout);
return -1;
}
/// Convert all raw PAs in the group into a single set of "cooked" PAs
/// under the leader.
/// @param[in] ca the object pointer
void
ca_coalesce(ca_o ca)
{
dict_t *dict_raw, *dict_cooked;
dnode_t *dnpr, *dnpc, *next;
dict_raw = ca->ca_raw_pa_dict;
assert(!ca->ca_cooked_pa_dict);
if ((dict_cooked = dict_create(DICTCOUNT_T_MAX, pa_cmp_by_pathname))) {
ca->ca_cooked_pa_dict = dict_cooked;
} else {
putil_syserr(2, "dict_create()");
}
for (dnpr = dict_first(dict_raw); dnpr;) {
pa_o raw_pa, ckd_pa;
next = dict_next(dict_raw, dnpr);
raw_pa = (pa_o)dnode_getkey(dnpr);
_ca_verbosity_pa(raw_pa, ca, "COALESCING");
// All data is in the key - that's why the value can be null.
if ((dnpc = dict_lookup(dict_cooked, raw_pa))) {
int keep_cooked = 0;
ckd_pa = (pa_o)dnode_getkey(dnpc);
if (!pa_is_read(raw_pa) && !pa_is_read(ckd_pa)) {
moment_s raw_timestamp, ckd_timestamp;
// If they're both destructive ops (non-read) then we
// need to consider timestamps and use the later one.
if (pa_has_timestamp(raw_pa) && pa_has_timestamp(ckd_pa)) {
// If the PA's have their own timestamps, use them.
raw_timestamp = pa_get_timestamp(raw_pa);
ckd_timestamp = pa_get_timestamp(ckd_pa);
} else {
// Otherwise key off the file times. This is for
// support of "dummy" PAs as used in shopping.
raw_timestamp = pa_get_moment(raw_pa);
ckd_timestamp = pa_get_moment(ckd_pa);
}
if (moment_cmp(raw_timestamp, ckd_timestamp, NULL) <= 0) {
// Cooked write op is newer and can stay.
keep_cooked = 1;
}
} else if (pa_is_read(raw_pa)) {
// There's no point replacing a read with another read,
// so regardless of whether the current cooked PA is a
// read or write, it can stay.
keep_cooked = 1;
} else {
// A write always beats a read.
}
if (!keep_cooked) {
dict_delete(dict_cooked, dnpc);
dnode_destroy(dnpc);
_ca_verbosity_pa(ckd_pa, ca, "REMOVING");
pa_destroy(ckd_pa);
if (!(dnpc = dnode_create(NULL))) {
putil_syserr(2, "dnode_create()");
}
ckd_pa = pa_copy(raw_pa);
dict_insert(dict_cooked, dnpc, ckd_pa);
}
} else {
if (!(dnpc = dnode_create(NULL))) {
putil_syserr(2, "dnode_create()");
}
ckd_pa = pa_copy(raw_pa);
dict_insert(dict_cooked, dnpc, ckd_pa);
}
// Clean up the raw set as we move PAs to the cooked one.
dict_delete(dict_raw, dnpr);
dnode_destroy(dnpr);
dnpr = next;
}
return;
}
int64_t compress(const char* in_filename, const char* out_filename, uint32_t dict_size, uint32_t ht_size, uint8_t flags) {
struct bitio *bd = bstdout;
struct dictionary *d = NULL;
struct stat file_stat;
time_t t;
FILE *fin = stdin;
char *md5_str;
int c, read_count = 0;
uint8_t bits, initial_bits;
uint32_t bitMask, cur, next_record, y;
uint64_t filesize = 0;
unsigned char *md5;
if (out_filename != NULL && in_filename != NULL && strcmp(in_filename, out_filename) == 0) {
errno = EINVAL;
goto error;
}
if (in_filename != NULL) {
fin = fopen(in_filename, "r");
if (fin == NULL)
goto error;
}
if (out_filename != NULL) {
bd = bitio_open(out_filename, 'w');
if (bd == NULL)
goto error;
}
//write metadata
if (flags & META_DICT_SIZE)
if (meta_write(bd, META_DICT_SIZE, &dict_size, sizeof(dict_size)) < 0)
goto error;
if (flags & META_MD5) {
if (fin != stdin) {
int md5_size;
md5 = compute_digest(fin, "md5", &md5_size);
if (meta_write(bd, META_MD5, md5, md5_size) < 0)
goto error;
md5_str = sprinth(md5, md5_size);
PRINT(1, "md5sum:\t\t\t%s\n", md5_str);
free(md5);
free(md5_str);
}
else
PRINT(1, "md5sum:\t\t\tNot availabe when reading from stdin\n");
}
if ((flags & META_NAME) && in_filename != NULL) { //don't put META_NAME if input = stdin
c = path_len(in_filename);
if (meta_write(bd, META_NAME, (void*)&in_filename[c], strlen(in_filename) - c + 1) < 0)
goto error;
}
if ((flags & META_TIMESTAMP) && in_filename != NULL) { //don't put META_TIMESTAMP if input = stdin
fstat(fileno(fin), &file_stat);
t = file_stat.st_mtime;
if (meta_write(bd, META_TIMESTAMP, &t, sizeof(t)) < 0)
goto error;
}
if (meta_finalize(bd) < 0)
goto error;
d = dict_new(dict_size, 1, ht_size, NUM_SYMBOLS);
if (d == NULL)
goto error;
next_record = dict_init(d);
initial_bits = 0;
bitMask = 1;
while (bitMask < next_record) {
bitMask <<= 1;
initial_bits++;
}
bits = initial_bits;
bitMask = 1 << bits;
cur = ROOT_NODE;
for(;;) {
c = fgetc(fin);
if (c == EOF) {
//emit last word
if (emit(bd, cur, bits) < 0)
goto error;
//emit EOF
dict_lookup(d, ROOT_NODE, EOF_SYMBOL, &y);
if (emit(bd, y, bits) < 0)
goto error;
break;
}
filesize++;
if (VERBOSE_LEVEL > 0 && ++read_count >= COUNT_THRESHOLD) {
read_count = 0;
PRINT(1, ".");
}
if (!dict_lookup(d, cur, (uint16_t) c, &y)) { //node not found
if (emit(bd, cur, bits) < 0)
goto error;
dict_fill(d, y, cur, (uint16_t) c, next_record++);
if (next_record & bitMask) {
bitMask <<= 1;
bits++;
}
if (next_record == dict_size) {
next_record = dict_reinit(d);
bits = initial_bits;
bitMask = 1 << bits;
}
// search again starting from last unmatched symbol
dict_lookup(d, ROOT_NODE, (uint16_t) c, &y);
}
cur = dict_next(d, y);
}
PRINT(1, "\nCompression Finished\n\n");
dict_delete(d);
bitio_flush(bd);
if (bd != bstdout)
bitio_close(bd);
if (fin != NULL)
fclose(fin);
return filesize;
error:
PRINT(1, "\n");
dict_delete(d);
bitio_flush(bd);
if (bd != bstdout)
bitio_close(bd);
if (fin != NULL)
fclose(fin);
return -1;
}
void dict_delete_free(dict_t *dict, dnode_t *node)
{
dict_delete(dict, node);
dict->freenode(node, dict->context);
}
void
track_cleanup(UNUSED_ARG(void *extra)) {
track_cfg.enabled = 0;
unreg_del_user_func(track_del_user, NULL);
dict_delete(track_db);
}
int
main(int argc, char **argv) {
char *merylCount = 0L;
char *fastaName = 0L;
int arg=1;
while (arg < argc) {
if (strcmp(argv[arg], "-m") == 0) {
merylCount = argv[++arg];
} else if (strcmp(argv[arg], "-f") == 0) {
fastaName = argv[++arg];
} else {
fprintf(stderr, "unknown option '%s'\n", argv[arg]);
}
arg++;
}
if ((merylCount == 0L) || (fastaName == 0L)) {
fprintf(stderr, "usage: %s -m <meryl-name-prefix> -f <fasta-file>\n", argv[0]);
exit(1);
}
// Open the count files
//
merylStreamReader *MSR = new merylStreamReader(merylCount);
fprintf(stderr, "Mers are "uint32FMT" bases.\n", MSR->merSize());
fprintf(stderr, "There are "uint64FMT" unique (copy = 1) mers.\n", MSR->numberOfUniqueMers());
fprintf(stderr, "There are "uint64FMT" distinct mers.\n", MSR->numberOfDistinctMers());
fprintf(stderr, "There are "uint64FMT" mers total.\n", MSR->numberOfTotalMers());
// Guess how many mers we can fit into 512MB, then report how many chunks we need to do.
uint32 merSize = MSR->merSize();
uint64 memoryLimit = 700 * 1024 * 1024;
uint64 perMer = sizeof(kMerLite) + sizeof(dnode_t);
uint64 mersPerBatch = memoryLimit / perMer;
uint32 numBatches = MSR->numberOfDistinctMers() / mersPerBatch;
uint32 batch = 0;
dnode_t *nodes = new dnode_t [mersPerBatch];
kMerLite *mers = new kMerLite [mersPerBatch];
if (MSR->numberOfDistinctMers() % mersPerBatch)
numBatches++;
fprintf(stderr, "perMer: "uint64FMT" bytes ("uint64FMT" for kMerLite, "uint64FMT" for dnode_t.\n",
perMer, (uint64)sizeof(kMerLite), (uint64)sizeof(dnode_t));
fprintf(stderr, "We can fit "uint64FMT" mers into "uint64FMT"MB.\n", mersPerBatch, memoryLimit >> 20);
fprintf(stderr, "So we need "uint32FMT" batches to verify the count.\n", numBatches);
while (MSR->validMer()) {
uint64 mersRemain = mersPerBatch;
dict_t *merDict = dict_create(mersPerBatch, kMerLiteSort);
batch++;
// STEP 1: Insert mersPerBatch into the merDict
//
fprintf(stderr, "STEP 1 BATCH "uint32FMTW(2)": Insert into merDict\n", batch);
while (MSR->nextMer() && mersRemain) {
mersRemain--;
mers[mersRemain] = MSR->theFMer();
// initialize the node with the value, then insert the node
// into the tree using the key
int32 val = (int32)MSR->theCount();
dnode_init(&nodes[mersRemain], (void *)val);
dict_insert(merDict, &nodes[mersRemain], &mers[mersRemain]);
}
// STEP 2: Stream the original file, decrementing the count
//
fprintf(stderr, "STEP 2 BATCH "uint32FMTW(2)": Stream fasta\n", batch);
seqStream *CS = new seqStream(fastaName, true);
merStream *MS = new merStream(new kMerBuilder(merSize), CS);
kMerLite mer;
dnode_t *nod;
while (MS->nextMer()) {
mer = MS->theFMer();
nod = dict_lookup(merDict, &mer);
if (nod != 0L) {
int32 val = (int32)dnode_get(nod);
val--;
dnode_put(nod, (void *)val);
} else {
// Unless the whole meryl file fit into our merDict, we cannot warn if
// we don't find mers.
//
if (numBatches == 1) {
char str[1024];
fprintf(stderr, "Didn't find node for mer '%s'\n", mer.merToString(merSize, str));
}
}
}
delete MS;
delete CS;
// STEP 3: Check every node in the tree to make sure that the counts
// are exactly zero.
//
fprintf(stderr, "STEP 3 BATCH "uint32FMTW(2)": Check\n", batch);
nod = dict_first(merDict);
while (nod) {
int32 val = (int32)dnode_get(nod);
kMerLite const *nodmer = (kMerLite const *)dnode_getkey(nod);
if (val != 0) {
char str[1024];
fprintf(stderr, "Got count "int32FMT" for mer '%s'\n",
val,
nodmer->merToString(merSize, str));
}
nod = dict_next(merDict, nod);
}
// STEP 4: Destroy the dictionary.
//
fprintf(stderr, "STEP 4 BATCH "uint32FMTW(2)": Destroy\n", batch);
while ((nod = dict_first(merDict)))
dict_delete(merDict, nod);
dict_destroy(merDict);
}
}
void lang_delete(lang* t) {
dict_map(t->map, free);
dict_delete(t->map);
free(t);
}
