END_TEST START_TEST(test_dict_add3) { TestData t, *pt; t.ivalue = 1; Dict *pdict = dict_create(); dict_add(pdict, SIZEOF("key1"), (char*)&t, sizeof(TestData)); pt = (TestData*)dict_get(pdict, SIZEOF("key1")); ck_assert_int_eq(pt->ivalue, 1); t.ivalue = 2; dict_add(pdict, SIZEOF("key1"), (char*)&t, sizeof(TestData)); pt = (TestData*)dict_get(pdict, SIZEOF("key1")); ck_assert_int_eq(pt->ivalue, 1); }
void _sql_init(void) { if (DBConnection < 0) { logging_register_module(sql); if (!_drivers) { _driver_mutex = mutex_create(); _drivers = dict_create(NULL); dict_set_key_type(_drivers, &type_str); dict_set_data_type(_drivers, &type_int); } if (ErrorSQL < 1) { exception_register(ErrorSQL); } typedescr_register_with_methods(DBConnection, dbconn_t); typedescr_register(DBTransaction, tx_t); } assert(DBConnection); }
void smpp_route_rebuild_database(SMPPServer *smpp_server) { info(0, "Rebuilding database routes"); SMPPRouting *smpp_routing = smpp_server->routing; List *inbound_routes = smpp_database_get_routes(smpp_server, SMPP_ROUTE_DIRECTION_INBOUND, NULL); /* Only inbound are built, outbound are built when ESME's connect */ List *old_inbound; Dict *old_outbound; gw_rwlock_wrlock(smpp_routing->lock); old_inbound = smpp_routing->inbound_routes; old_outbound = smpp_routing->outbound_routes; smpp_routing->inbound_routes = inbound_routes; smpp_routing->outbound_routes = dict_create(1024, (void(*)(void *))smpp_outbound_routes_destroy); /* Just reset, they will repopulate on their own */ gw_rwlock_unlock(smpp_routing->lock); gwlist_destroy(old_inbound, (void(*)(void *))smpp_route_destroy); dict_destroy(old_outbound); }
int cxml_node_addnode(CLOG_INFO* info, CXMLNODE *parent, CXMLNODE *child) { int return_value=0; assert(NULL!=parent); assert(NULL!=child); // if the sub node is NULL create a new sub table if(NULL==parent->sub) { clog( info, CTRACE, "XML: xml_node_addnode(), creating sub node"); parent->sub = dict_create(DICTCOUNT_T_MAX, (dict_comp_t)strcmp); // allow duplicates dict_allow_dupes(parent->sub); } // if the previous stuff worked, add the name/value pair if(NULL!=parent->sub && NULL!=child->name && NULL!=child->name->string) { dnode_t *node = (parent->sub)->dict_allocnode((parent->sub)->dict_context); char *key = strdup(child->name->string); clog( info, CTRACE, "XML: xml_node_adddata(), Saving sub node name=\"%.80s\"", child->name->string); if (node) { dnode_init(node, child); dict_insert(parent->sub, node, key); // let the child know who they are child->me = node; // success! return_value=1; } } // let the child know whow their parent is if(NULL!=parent && NULL!=child) { child->parent = parent; } return return_value; }
static int tolua_dict_set_usertype(lua_State * L, int type) { dict self = (dict)tolua_tousertype(L, 1, 0); const char *name = tolua_tostring(L, 2, 0); unit *value = tolua_tousertype(L, 3, 0); attrib *a = a_find(*self, &at_dict); variant val; val.v = value; for (; a && a->type == &at_dict; a = a->next) { if (strcmp(dict_name(a), name) == 0) { dict_set(a, type, val); return 0; } } a = a_add(self, dict_create(name, type, val)); return 0; }
int init_update(void) { dict_types type; memset(&type, 0, sizeof(type)); type.hash_function = update_dict_hash_function; type.key_compare = update_dict_key_compare; type.key_dup = update_dict_key_dup; type.key_destructor = update_dict_key_free; type.val_dup = update_dict_val_dup; type.val_destructor = update_dict_val_free; dict_update = dict_create(&type, 64); if (dict_update == NULL) return -__LINE__; nw_timer_set(&timer, 60, true, on_timer, NULL); nw_timer_start(&timer); return 0; }
static int tolua_dict_set_number(lua_State * L) { dict self = (dict)tolua_tousertype(L, 1, 0); const char *name = tolua_tostring(L, 2, 0); lua_Number value = tolua_tonumber(L, 3, 0); attrib *a = a_find(*self, &at_dict); variant val; val.f = (float)value; for (; a && a->type == &at_dict; a = a->next) { if (strcmp(dict_name(a), name) == 0) { dict_set(a, TREAL, val); return 0; } } a = a_add(self, dict_create(name, TREAL, val)); return 0; }
END_TEST START_TEST(test_dict_add1) { TestData t; t.ivalue = 1; Dict *pdict = dict_create(); dict_add(pdict, SIZEOF("key1"), (char*)&t, sizeof(TestData)); ck_assert_int_eq(dict_get_count(pdict), 1); TestData *t1 = (TestData*)dict_get(pdict, SIZEOF("key1")); ck_assert_msg(t1 != &t, "dict did not make a copy of input data"); ck_assert_int_eq(t1->ivalue, 1); dict_add(pdict, SIZEOF("key2"), (char*)&t, sizeof(TestData)); TestData *t2 = (TestData*)dict_get(pdict, SIZEOF("key2")); ck_assert_msg(t1 != t2, "cannot add the same data with two different keys"); dict_free(pdict); }
void testBasicDict() { struct dict *d = dict_create(); dict_set(d, "name", "drmaa2"); dict_del(d, "name"); dict_set(d, "name", "drmaa2"); assert(dict_has(d, "name") != 0); assert(dict_has(d, "language") == 0); const char *v = dict_get(d, "name"); assert(strcmp(v, "drmaa2") == 0); dict_set(d, "language", "c"); assert(dict_has(d, "language") != 0); dict_del(d, "name"); assert(dict_has(d, "name") == 0); v = dict_get(d, "language"); assert(strcmp(v, "c") == 0); dict_free(d); }
void cmd_init() { command_list = dict_create(); dict_set_free_funcs(command_list, NULL, (dict_free_f *)cmd_free_subcmds); // Initialize our own commands cmd_register_list(commands, NULL); cmd_alias("exit", "quit", NULL); // Initialize external commands cmd_server_init(); cmd_link_init(); cmd_conf_init(); cmd_oper_init(); cmd_service_init(); cmd_forward_init(); cmd_feature_init(); cmd_jupe_init(); cmd_class_init(); cmd_pseudo_init(); cmd_client_init(); cmd_webirc_init(); }
size_t get_file(const char *file_path, struct dict **dicts) { char *token, buffer[1024]; size_t size = 0, alloc = 1; struct dict *d = malloc(sizeof(struct dict)); FILE *f = fopen(file_path, "r"); while (fgets(buffer, sizeof(buffer), f) != NULL) { if ((token = strchr(buffer, '#')) != NULL) *token = '\0'; if ((token = strchr(buffer, '=')) == NULL) continue; if (size == alloc) { alloc *= 2; d = realloc(d, alloc * sizeof(struct dict)); } *token++ = '\0'; while (*token == ' ' || *token == '\t') token++; dict_create(d + size++, buffer, strlen(buffer), token, strlen(token)); } fclose(f); return size; }
TEST(mydict, dict) { dict_t* d = dict_create((void*)123); int k, v; dict_node_t* node = NULL; int i = 0; int sum = 0; for (i = 0; i < 10000; i++) { k = i, v = i; node = dict_add(d, &k, sizeof(k), &v, sizeof(v)); ASSERT_TRUE(node != NULL); sum += i; } for (i = 0; i < 10000; i++) { node = dict_find(d, &k, sizeof(k)); ASSERT_EQ((long)dict_get_val(node), i); } i = 0; for(node = dict_first(d); node; node = dict_next(d, node)) { i += (long)dict_get_val(node); } ASSERT_EQ(i, sum); for (i = 0; i < 10000; i++) { k = i; node = dict_delete(d, &k, sizeof(k)); ASSERT_EQ((long)dict_get_val(node), i); } ASSERT_TRUE(dict_empty(d)); dict_destroy(d); }
int store_file_init(const Octstr *fname, long dump_freq) { /* Initialize function pointers */ store_messages = store_file_messages; store_save = store_file_save; store_save_ack = store_file_save_ack; store_load = store_file_load; store_dump = store_file_dump; store_shutdown = store_file_shutdown; store_status = store_file_status; if (fname == NULL) return 0; /* we are done */ if (octstr_len(fname) > (FILENAME_MAX-5)) panic(0, "Store file filename too long: `%s', failed to init.", octstr_get_cstr(fname)); filename = octstr_duplicate(fname); newfile = octstr_format("%s.new", octstr_get_cstr(filename)); bakfile = octstr_format("%s.bak", octstr_get_cstr(filename)); sms_dict = dict_create(1024, msg_destroy_item); if (dump_freq > 0) dump_frequency = dump_freq; else dump_frequency = BB_STORE_DEFAULT_DUMP_FREQ; file_mutex = mutex_create(); active = 1; loaded = gwlist_create(); gwlist_add_producer(loaded); return 0; }
/* Set the clip according to the currently selected range in the data sheet */ static void data_sheet_set_clip (PsppireSheet *sheet) { int i; struct casewriter *writer ; PsppireSheetRange range; PsppireDataStore *ds; struct case_map *map = NULL; casenumber max_rows; size_t max_columns; gint row0, rowi; gint col0, coli; ds = PSPPIRE_DATA_STORE (psppire_sheet_get_model (sheet)); psppire_sheet_get_selected_range (sheet, &range); col0 = MIN (range.col0, range.coli); coli = MAX (range.col0, range.coli); row0 = MIN (range.row0, range.rowi); rowi = MAX (range.row0, range.rowi); /* If nothing selected, then use active cell */ if ( row0 < 0 || col0 < 0 ) { gint row, col; psppire_sheet_get_active_cell (sheet, &row, &col); row0 = rowi = row; col0 = coli = col; } /* The sheet range can include cells that do not include data. Exclude them from the range. */ max_rows = psppire_data_store_get_case_count (ds); if (rowi >= max_rows) { if (max_rows == 0) return; rowi = max_rows - 1; } max_columns = dict_get_var_cnt (ds->dict->dict); if (coli >= max_columns) { if (max_columns == 0) return; coli = max_columns - 1; } /* Destroy any existing clip */ if ( clip_datasheet ) { casereader_destroy (clip_datasheet); clip_datasheet = NULL; } if ( clip_dict ) { dict_destroy (clip_dict); clip_dict = NULL; } /* Construct clip dictionary. */ clip_dict = dict_create (dict_get_encoding (ds->dict->dict)); for (i = col0; i <= coli; i++) dict_clone_var_assert (clip_dict, dict_get_var (ds->dict->dict, i)); /* Construct clip data. */ map = case_map_by_name (ds->dict->dict, clip_dict); writer = autopaging_writer_create (dict_get_proto (clip_dict)); for (i = row0; i <= rowi ; ++i ) { struct ccase *old = psppire_data_store_get_case (ds, i); if (old != NULL) casewriter_write (writer, case_map_execute (map, old)); else casewriter_force_error (writer); } case_map_destroy (map); clip_datasheet = casewriter_make_reader (writer); data_sheet_update_clipboard (sheet); }
int main(int argc,char *argv[],char *env[]) { pdict = dict_create(); FILE *fp = fopen(FILE_NAME,"r"); if(fp == NULL) { perror("fopen"); return FAILED; } char english[1000]; char chinese[1000]; while(!feof(fp)) { memset(english,0,sizeof(english)); memset(chinese,0,sizeof(chinese)); if(fgets(english,sizeof(english),fp) == NULL) break; *strchr(english,'\n') = 0; if(fgets(chinese,sizeof(chinese),fp) == NULL) break; *strchr(chinese,'\n') = 0; dict_insert(pdict,english,chinese); } signal(28,handle); show_window(); move_xy(3,15); char key; while(1) { key = get_key(); switch(key) { case ESC: clear_screen(); move_xy(1,1); return 0; case BACKSPACE: if(buf_i > 0) { move_left(1); putchar(' '); fflush(stdout); move_left(1); buf_i--; input_buf[buf_i] = 0; show_word(pdict,input_buf); } break; default: if(buf_i <= win_y - 20 && isprint(key)) { putchar(key); fflush(stdout); input_buf[buf_i] = key; buf_i++; input_buf[buf_i] = 0; show_word(pdict,input_buf); } } } return 0; }
static void run_smsbox(void *arg) { Boxc *newconn; long sender; Msg *msg; List *keys; Octstr *key; gwlist_add_producer(flow_threads); newconn = arg; newconn->incoming = gwlist_create(); gwlist_add_producer(newconn->incoming); newconn->retry = incoming_sms; newconn->outgoing = outgoing_sms; newconn->sent = dict_create(smsbox_max_pending, NULL); newconn->pending = semaphore_create(smsbox_max_pending); sender = gwthread_create(boxc_sender, newconn); if (sender == -1) { error(0, "Failed to start a new thread, disconnecting client <%s>", octstr_get_cstr(newconn->client_ip)); goto cleanup; } /* * We register newconn in the smsbox_list here but mark newconn as routable * after identification or first message received from smsbox. So we can avoid * a race condition for routable smsboxes (otherwise between startup and * registration we will forward some messages to smsbox). */ gw_rwlock_wrlock(smsbox_list_rwlock); gwlist_append(smsbox_list, newconn); gw_rwlock_unlock(smsbox_list_rwlock); gwlist_add_producer(newconn->outgoing); boxc_receiver(newconn); gwlist_remove_producer(newconn->outgoing); /* remove us from smsbox routing list */ gw_rwlock_wrlock(smsbox_list_rwlock); gwlist_delete_equal(smsbox_list, newconn); if (newconn->boxc_id) { dict_remove(smsbox_by_id, newconn->boxc_id); } gw_rwlock_unlock(smsbox_list_rwlock); /* * check if we in the shutdown phase and sms dequeueing thread * has removed the producer already */ if (gwlist_producer_count(newconn->incoming) > 0) gwlist_remove_producer(newconn->incoming); /* check if we are still waiting for ack's and semaphore locked */ if (dict_key_count(newconn->sent) >= smsbox_max_pending) semaphore_up(newconn->pending); /* allow sender to go down */ gwthread_join(sender); /* put not acked msgs into incoming queue */ keys = dict_keys(newconn->sent); while((key = gwlist_extract_first(keys)) != NULL) { msg = dict_remove(newconn->sent, key); gwlist_produce(incoming_sms, msg); octstr_destroy(key); } gw_assert(gwlist_len(keys) == 0); gwlist_destroy(keys, octstr_destroy_item); /* clear our send queue */ while((msg = gwlist_extract_first(newconn->incoming)) != NULL) { gwlist_produce(incoming_sms, msg); } cleanup: gw_assert(gwlist_len(newconn->incoming) == 0); gwlist_destroy(newconn->incoming, NULL); gw_assert(dict_key_count(newconn->sent) == 0); dict_destroy(newconn->sent); semaphore_destroy(newconn->pending); boxc_destroy(newconn); /* wakeup the dequeueing thread */ gwthread_wakeup(sms_dequeue_thread); gwlist_remove_producer(flow_threads); }
int smpp_pdu_init(Cfg *cfg) { CfgGroup *grp; List *l; if (initialized) return 0; l = cfg_get_multi_group(cfg, octstr_imm("smpp-tlv")); tlvs = gwlist_create(); tlvs_by_tag = dict_create(1024, (void(*)(void*))dict_destroy); tlvs_by_name = dict_create(1024, (void(*)(void*))dict_destroy); while (l != NULL && (grp = gwlist_extract_first(l)) != NULL) { struct smpp_tlv *tlv; Octstr *tmp, *smsc_id; List *l2; tlv = gw_malloc(sizeof(*tlv)); if ((tlv->name = cfg_get(grp, octstr_imm("name"))) == NULL) { error(0, "SMPP: Unable to get name for smpp-tlv."); smpp_tlv_destroy(tlv); goto failed; } if (cfg_get_integer(&tlv->tag, grp, octstr_imm("tag")) == -1) { error(0, "SMPP: Unable to get tag for smpp-tlv."); smpp_tlv_destroy(tlv); goto failed; } if (cfg_get_integer(&tlv->length, grp, octstr_imm("length")) == -1) { error(0, "SMPP: Unable to get length for smpp-tlv."); smpp_tlv_destroy(tlv); goto failed; } if ((tmp = cfg_get(grp, octstr_imm("type"))) == NULL) { error(0, "SMPP: Unable to get type for smpp-tlv."); smpp_tlv_destroy(tlv); goto failed; } if (octstr_str_case_compare(tmp, "octetstring") == 0) tlv->type = SMPP_TLV_OCTETS; else if (octstr_str_case_compare(tmp, "nulterminated") == 0) tlv->type = SMPP_TLV_NULTERMINATED; else if (octstr_str_case_compare(tmp, "integer") == 0) tlv->type = SMPP_TLV_INTEGER; else { error(0, "SMPP: Unknown type for smpp-tlv: `%s'", octstr_get_cstr(tmp)); octstr_destroy(tmp); smpp_tlv_destroy(tlv); goto failed; } octstr_destroy(tmp); /* put to all TLVs */ gwlist_produce(tlvs, tlv); smsc_id = cfg_get(grp, octstr_imm("smsc-id")); if (smsc_id != NULL) { l2 = octstr_split(smsc_id, octstr_imm(";")); octstr_destroy(smsc_id); } else { l2 = gwlist_create(); gwlist_produce(l2, octstr_create(DEFAULT_SMSC_ID)); } while(l2 != NULL && (smsc_id = gwlist_extract_first(l2)) != NULL) { Dict *tmp_dict; debug("sms.smpp", 0, "adding smpp-tlv for smsc-id=%s", octstr_get_cstr(smsc_id)); tmp_dict = dict_get(tlvs_by_name, smsc_id); if (tmp_dict == NULL) { tmp_dict = dict_create(1024, NULL); dict_put(tlvs_by_name, smsc_id, tmp_dict); } /* put into dict */ if (!dict_put_once(tmp_dict, tlv->name, tlv)) { error(0, "SMPP: Double TLV name %s found.", octstr_get_cstr(tlv->name)); octstr_destroy(smsc_id); goto failed; } tmp_dict = dict_get(tlvs_by_tag, smsc_id); if (tmp_dict == NULL) { tmp_dict = dict_create(1024, NULL); dict_put(tlvs_by_tag, smsc_id, tmp_dict); } tmp = octstr_format("%ld", tlv->tag); if (!dict_put_once(tmp_dict, tmp, tlv)) { error(0, "SMPP: Double TLV tag %s found.", octstr_get_cstr(tmp)); gwlist_destroy(l2, octstr_destroy_item); octstr_destroy(tmp); octstr_destroy(smsc_id); goto failed; } octstr_destroy(tmp); octstr_destroy(smsc_id); } gwlist_destroy(l2, octstr_destroy_item); } gwlist_destroy(l, NULL); initialized = 1; return 0; failed: gwlist_destroy(tlvs, (void(*)(void*))smpp_tlv_destroy); dict_destroy(tlvs_by_tag); dict_destroy(tlvs_by_name); return -1; }
mCfg *mms_cfg_read(Octstr *file) { Octstr *sf; List *lines; int i, n; mCfg *cfg; mCfgGrp *grp = NULL; int skip = 0; gw_assert(file); if ((sf = octstr_read_file(octstr_get_cstr(file))) == NULL) { mms_error(errno, "mms_cfg", NULL, "failed to read config from `%s'", octstr_get_cstr(file)); return NULL; } cfg = gw_malloc(sizeof *cfg); cfg->file = octstr_duplicate(file); cfg->grps = dict_create(7, NULL); cfg->xcfg = NULL; cfg->cfg_funcs = NULL; lines = octstr_split(sf, octstr_imm("\n")); for (i = 0, n = gwlist_len(lines); i < n; i++) { Octstr *current = gwlist_get(lines,i); int pos; octstr_strip_blanks(current); if (octstr_len(current) == 0) { /* end of group. */ grp = NULL; skip = 0; continue; } else if (skip || octstr_get_char(current, 0) == '#') continue; if ((pos = octstr_search_char(current, '=',0)) > 0) { /* a field name. first see if start of grp */ Octstr *field = octstr_copy(current,0,pos); Octstr *value = octstr_copy(current,pos+1,octstr_len(current)); octstr_strip_blanks(field); fixup_value(value, i+1); #if 0 mms_info(0, "mms_cfg", NULL, "field/value: [%s - %s]", octstr_get_cstr(field), octstr_get_cstr(value)); #endif if (octstr_str_case_compare(field, "group") == 0) if (grp == NULL) { /* grp name. */ int ismulti = is_multigroup(value); if (ismulti < 0) { mms_info(0, "mms_cfg", NULL, "Skipping unknown group `%s' at line %d of conf file", octstr_get_cstr(value), i+1); skip = 1; } else { grp = gw_malloc(sizeof *grp); grp->name = octstr_duplicate(value); grp->fields = dict_create(23, (void (*)(void *))octstr_destroy); if (ismulti) { List *l = dict_get(cfg->grps, value); if (l == NULL) { l = gwlist_create(); dict_put(cfg->grps, value, l); } gwlist_append(l, grp); } else if (dict_put_once(cfg->grps, value, grp) == 0) panic(0, "Group `%s' [at line %d] cannot appear more " "than once in config!", octstr_get_cstr(value), i+1); } } else panic(0,"`group' is an illegal field name " "within a group at line %d in config file!", i+1); else if (grp) /* an ordinary field name. */ check_and_add_field(grp, field, value,i+1); else panic(0, "A group must begin with a `group = group_name' " "clause [at line %d in config file]", i+1); octstr_destroy(field); octstr_destroy(value); } else panic(0, "mal-formed entry in conf file at line %d!", i+1); } gwlist_destroy(lines, (gwlist_item_destructor_t *)octstr_destroy); octstr_destroy(sf); /* Now check if config-source is set, use that. */ if ((grp = mms_cfg_get_single(cfg, octstr_imm("config-source"))) != NULL) { Octstr *init = mms_cfg_get(cfg, grp, octstr_imm("config-library-init-param")); cfg->cfg_funcs = _mms_load_module(cfg, grp, "config-library", "cfg_funcs", NULL); if (cfg->cfg_funcs == NULL || cfg->cfg_funcs->read == NULL || (cfg->xcfg = cfg->cfg_funcs->read(init)) == NULL) { mms_error(0, "mms_cfg", NULL, "Failed to load cfg reader library from conf!"); mms_cfg_destroy(cfg); cfg = NULL; } octstr_destroy(init); } return cfg; }
static int combine_files (enum comb_command_type command, struct lexer *lexer, struct dataset *ds) { struct comb_proc proc; bool saw_by = false; bool saw_sort = false; struct casereader *active_file = NULL; char *first_name = NULL; char *last_name = NULL; struct taint *taint = NULL; size_t n_tables = 0; size_t allocated_files = 0; size_t i; proc.files = NULL; proc.n_files = 0; proc.dict = dict_create (get_default_encoding ()); proc.output = NULL; proc.matcher = NULL; subcase_init_empty (&proc.by_vars); proc.first = NULL; proc.last = NULL; proc.buffered_case = NULL; proc.prev_BY = NULL; dict_set_case_limit (proc.dict, dict_get_case_limit (dataset_dict (ds))); lex_match (lexer, T_SLASH); for (;;) { struct comb_file *file; enum comb_file_type type; if (lex_match_id (lexer, "FILE")) type = COMB_FILE; else if (command == COMB_MATCH && lex_match_id (lexer, "TABLE")) { type = COMB_TABLE; n_tables++; } else break; lex_match (lexer, T_EQUALS); if (proc.n_files >= allocated_files) proc.files = x2nrealloc (proc.files, &allocated_files, sizeof *proc.files); file = &proc.files[proc.n_files++]; file->type = type; subcase_init_empty (&file->by_vars); subcase_init_empty (&file->src); subcase_init_empty (&file->dst); file->mv = NULL; file->handle = NULL; file->dict = NULL; file->reader = NULL; file->data = NULL; file->is_sorted = true; file->in_name = NULL; file->in_var = NULL; if (lex_match (lexer, T_ASTERISK)) { if (!dataset_has_source (ds)) { msg (SE, _("Cannot specify the active dataset since none " "has been defined.")); goto error; } if (proc_make_temporary_transformations_permanent (ds)) msg (SE, _("This command may not be used after TEMPORARY when " "the active dataset is an input source. " "Temporary transformations will be made permanent.")); file->dict = dict_clone (dataset_dict (ds)); } else { file->handle = fh_parse (lexer, FH_REF_FILE, dataset_session (ds)); if (file->handle == NULL) goto error; file->reader = any_reader_open (file->handle, NULL, &file->dict); if (file->reader == NULL) goto error; } while (lex_match (lexer, T_SLASH)) if (lex_match_id (lexer, "RENAME")) { if (!parse_dict_rename (lexer, file->dict)) goto error; } else if (lex_match_id (lexer, "IN")) { lex_match (lexer, T_EQUALS); if (lex_token (lexer) != T_ID) { lex_error (lexer, NULL); goto error; } if (file->in_name) { msg (SE, _("Multiple IN subcommands for a single FILE or " "TABLE.")); goto error; } file->in_name = xstrdup (lex_tokcstr (lexer)); lex_get (lexer); } else if (lex_match_id (lexer, "SORT")) { file->is_sorted = false; saw_sort = true; } if (!merge_dictionary (proc.dict, file)) goto error; } while (lex_token (lexer) != T_ENDCMD) { if (lex_match (lexer, T_BY)) { const struct variable **by_vars; size_t i; bool ok; if (saw_by) { lex_sbc_only_once ("BY"); goto error; } saw_by = true; lex_match (lexer, T_EQUALS); if (!parse_sort_criteria (lexer, proc.dict, &proc.by_vars, &by_vars, NULL)) goto error; ok = true; for (i = 0; i < proc.n_files; i++) { struct comb_file *file = &proc.files[i]; size_t j; for (j = 0; j < subcase_get_n_fields (&proc.by_vars); j++) { const char *name = var_get_name (by_vars[j]); struct variable *var = dict_lookup_var (file->dict, name); if (var != NULL) subcase_add_var (&file->by_vars, var, subcase_get_direction (&proc.by_vars, j)); else { if (file->handle != NULL) msg (SE, _("File %s lacks BY variable %s."), fh_get_name (file->handle), name); else msg (SE, _("Active dataset lacks BY variable %s."), name); ok = false; } } assert (!ok || subcase_conformable (&file->by_vars, &proc.files[0].by_vars)); } free (by_vars); if (!ok) goto error; } else if (command != COMB_UPDATE && lex_match_id (lexer, "FIRST")) { if (first_name != NULL) { lex_sbc_only_once ("FIRST"); goto error; } lex_match (lexer, T_EQUALS); if (!lex_force_id (lexer)) goto error; first_name = xstrdup (lex_tokcstr (lexer)); lex_get (lexer); } else if (command != COMB_UPDATE && lex_match_id (lexer, "LAST")) { if (last_name != NULL) { lex_sbc_only_once ("LAST"); goto error; } lex_match (lexer, T_EQUALS); if (!lex_force_id (lexer)) goto error; last_name = xstrdup (lex_tokcstr (lexer)); lex_get (lexer); } else if (lex_match_id (lexer, "MAP")) { /* FIXME. */ } else if (lex_match_id (lexer, "DROP")) { if (!parse_dict_drop (lexer, proc.dict)) goto error; } else if (lex_match_id (lexer, "KEEP")) { if (!parse_dict_keep (lexer, proc.dict)) goto error; } else { lex_error (lexer, NULL); goto error; } if (!lex_match (lexer, T_SLASH) && lex_token (lexer) != T_ENDCMD) { lex_end_of_command (lexer); goto error; } } if (!saw_by) { if (command == COMB_UPDATE) { lex_sbc_missing ("BY"); goto error; } if (n_tables) { msg (SE, _("BY is required when %s is specified."), "TABLE"); goto error; } if (saw_sort) { msg (SE, _("BY is required when %s is specified."), "SORT"); goto error; } } /* Add IN, FIRST, and LAST variables to master dictionary. */ for (i = 0; i < proc.n_files; i++) { struct comb_file *file = &proc.files[i]; if (!create_flag_var ("IN", file->in_name, proc.dict, &file->in_var)) goto error; } if (!create_flag_var ("FIRST", first_name, proc.dict, &proc.first) || !create_flag_var ("LAST", last_name, proc.dict, &proc.last)) goto error; dict_delete_scratch_vars (proc.dict); dict_compact_values (proc.dict); /* Set up mapping from each file's variables to master variables. */ for (i = 0; i < proc.n_files; i++) { struct comb_file *file = &proc.files[i]; size_t src_var_cnt = dict_get_var_cnt (file->dict); size_t j; file->mv = xnmalloc (src_var_cnt, sizeof *file->mv); for (j = 0; j < src_var_cnt; j++) { struct variable *src_var = dict_get_var (file->dict, j); struct variable *dst_var = dict_lookup_var (proc.dict, var_get_name (src_var)); if (dst_var != NULL) { size_t n = subcase_get_n_fields (&file->src); file->mv[n] = var_get_missing_values (src_var); subcase_add_var (&file->src, src_var, SC_ASCEND); subcase_add_var (&file->dst, dst_var, SC_ASCEND); } } } proc.output = autopaging_writer_create (dict_get_proto (proc.dict)); taint = taint_clone (casewriter_get_taint (proc.output)); /* Set up case matcher. */ proc.matcher = case_matcher_create (); for (i = 0; i < proc.n_files; i++) { struct comb_file *file = &proc.files[i]; if (file->reader == NULL) { if (active_file == NULL) { proc_discard_output (ds); file->reader = active_file = proc_open_filtering (ds, false); } else file->reader = casereader_clone (active_file); } if (!file->is_sorted) file->reader = sort_execute (file->reader, &file->by_vars); taint_propagate (casereader_get_taint (file->reader), taint); file->data = casereader_read (file->reader); if (file->type == COMB_FILE) case_matcher_add_input (proc.matcher, &file->by_vars, &file->data, &file->is_minimal); } if (command == COMB_ADD) execute_add_files (&proc); else if (command == COMB_MATCH) execute_match_files (&proc); else if (command == COMB_UPDATE) execute_update (&proc); else NOT_REACHED (); case_matcher_destroy (proc.matcher); proc.matcher = NULL; close_all_comb_files (&proc); if (active_file != NULL) proc_commit (ds); dataset_set_dict (ds, proc.dict); dataset_set_source (ds, casewriter_make_reader (proc.output)); proc.dict = NULL; proc.output = NULL; free_comb_proc (&proc); free (first_name); free (last_name); return taint_destroy (taint) ? CMD_SUCCESS : CMD_CASCADING_FAILURE; error: if (active_file != NULL) proc_commit (ds); free_comb_proc (&proc); taint_destroy (taint); free (first_name); free (last_name); return CMD_CASCADING_FAILURE; }
/************************************************************************* * dict_import: read in an ASCII dictionary. * * dict_fname - name of dictionary file * parameters to create a DICTIONARY structure (see dict_create) * * Returns: pointer to created DICTIONARY structure * (NULL on failure) * *************************************************************************/ DICTIONARY *dict_import( const char *dict_fname , const long initial_string_count , const long initial_hash_entries , const long max_chain_length ) { DICTIONARY *dict; char buffer[BUFLEN], ch; int index, c, c0; long number; FILE *fi = NULL; /*********** ** Dictionary setup. ***********/ dict = dict_create( 4, initial_string_count , initial_hash_entries , max_chain_length ); if ( dict == NULL ) goto err_exit; /*********** ** Read the dictionary file ** Each line should have one word or a string delimited by '|' ***********/ if ( (fi=fopen(dict_fname,"r")) == NULL ) goto err_exit; while( fgets(buffer,BUFLEN,fi) != NULL ) { c0 = 0; /* Skip to non-blank */ while ( (c0<BUFLEN-2) && (buffer[c0]==' ') ) ++c0; if ( buffer[c0] == '|' ) { c = ++c0; ch = '|'; } else { c = c0; ch = ' '; } /* endif */ /* Scan to blank or matching '|' */ while ( (c<BUFLEN-1) && (buffer[c]!='\0') && (buffer[c]!='\n') && (buffer[c]!=ch) ) ++c; buffer[c] = '\0'; /* Insert the word */ if ( dict_insert(dict,buffer+c0,1,0,NULL,&number) == NULL ) goto err_exit; } /* endwhile */ /*********** ** Fill in the dictionary parameter vector. ***********/ if ( dict_set_parm_values(dict) == FALSE ) goto err_exit; /*********** ** Update the table of contents for HASH STTB STAR ***********/ if ( (index=dict_toc_index(dict,"HASH")) == -1 ) goto err_exit; dict->toc[index].size = dict->table_size * sizeof(long); if ( (index=dict_toc_index(dict,"STTB")) == -1 ) goto err_exit; dict->toc[index].size = dict->string_max * sizeof(char); if ( (index=dict_toc_index(dict,"STAR")) == -1 ) goto err_exit; dict->toc[index].size = dict->array_size * sizeof(STRING_ENTRY); /* Success. Return a pointer to the new dictionary. */ fclose(fi); return( dict ); /* Failure. Ignominiously erase our tracks and return NULL. */ err_exit: if ( fi != NULL ) fclose(fi); dict_destroy( dict ); return NULL; }
dict_t * minerva_dict_create(void) { return dict_create(); }
int main(int argc, char **argv) { char *merylCount = 0L; char *fastaName = 0L; int arg=1; while (arg < argc) { if (strcmp(argv[arg], "-m") == 0) { merylCount = argv[++arg]; } else if (strcmp(argv[arg], "-f") == 0) { fastaName = argv[++arg]; } else { fprintf(stderr, "unknown option '%s'\n", argv[arg]); } arg++; } if ((merylCount == 0L) || (fastaName == 0L)) { fprintf(stderr, "usage: %s -m <meryl-name-prefix> -f <fasta-file>\n", argv[0]); exit(1); } // Open the count files // merylStreamReader *MSR = new merylStreamReader(merylCount); fprintf(stderr, "Mers are "uint32FMT" bases.\n", MSR->merSize()); fprintf(stderr, "There are "uint64FMT" unique (copy = 1) mers.\n", MSR->numberOfUniqueMers()); fprintf(stderr, "There are "uint64FMT" distinct mers.\n", MSR->numberOfDistinctMers()); fprintf(stderr, "There are "uint64FMT" mers total.\n", MSR->numberOfTotalMers()); // Guess how many mers we can fit into 512MB, then report how many chunks we need to do. uint32 merSize = MSR->merSize(); uint64 memoryLimit = 700 * 1024 * 1024; uint64 perMer = sizeof(kMerLite) + sizeof(dnode_t); uint64 mersPerBatch = memoryLimit / perMer; uint32 numBatches = MSR->numberOfDistinctMers() / mersPerBatch; uint32 batch = 0; dnode_t *nodes = new dnode_t [mersPerBatch]; kMerLite *mers = new kMerLite [mersPerBatch]; if (MSR->numberOfDistinctMers() % mersPerBatch) numBatches++; fprintf(stderr, "perMer: "uint64FMT" bytes ("uint64FMT" for kMerLite, "uint64FMT" for dnode_t.\n", perMer, (uint64)sizeof(kMerLite), (uint64)sizeof(dnode_t)); fprintf(stderr, "We can fit "uint64FMT" mers into "uint64FMT"MB.\n", mersPerBatch, memoryLimit >> 20); fprintf(stderr, "So we need "uint32FMT" batches to verify the count.\n", numBatches); while (MSR->validMer()) { uint64 mersRemain = mersPerBatch; dict_t *merDict = dict_create(mersPerBatch, kMerLiteSort); batch++; // STEP 1: Insert mersPerBatch into the merDict // fprintf(stderr, "STEP 1 BATCH "uint32FMTW(2)": Insert into merDict\n", batch); while (MSR->nextMer() && mersRemain) { mersRemain--; mers[mersRemain] = MSR->theFMer(); // initialize the node with the value, then insert the node // into the tree using the key int32 val = (int32)MSR->theCount(); dnode_init(&nodes[mersRemain], (void *)val); dict_insert(merDict, &nodes[mersRemain], &mers[mersRemain]); } // STEP 2: Stream the original file, decrementing the count // fprintf(stderr, "STEP 2 BATCH "uint32FMTW(2)": Stream fasta\n", batch); seqStream *CS = new seqStream(fastaName, true); merStream *MS = new merStream(new kMerBuilder(merSize), CS); kMerLite mer; dnode_t *nod; while (MS->nextMer()) { mer = MS->theFMer(); nod = dict_lookup(merDict, &mer); if (nod != 0L) { int32 val = (int32)dnode_get(nod); val--; dnode_put(nod, (void *)val); } else { // Unless the whole meryl file fit into our merDict, we cannot warn if // we don't find mers. // if (numBatches == 1) { char str[1024]; fprintf(stderr, "Didn't find node for mer '%s'\n", mer.merToString(merSize, str)); } } } delete MS; delete CS; // STEP 3: Check every node in the tree to make sure that the counts // are exactly zero. // fprintf(stderr, "STEP 3 BATCH "uint32FMTW(2)": Check\n", batch); nod = dict_first(merDict); while (nod) { int32 val = (int32)dnode_get(nod); kMerLite const *nodmer = (kMerLite const *)dnode_getkey(nod); if (val != 0) { char str[1024]; fprintf(stderr, "Got count "int32FMT" for mer '%s'\n", val, nodmer->merToString(merSize, str)); } nod = dict_next(merDict, nod); } // STEP 4: Destroy the dictionary. // fprintf(stderr, "STEP 4 BATCH "uint32FMTW(2)": Destroy\n", batch); while ((nod = dict_first(merDict))) dict_delete(merDict, nod); dict_destroy(merDict); } }
int main(int argc, char **argv) { uint32 pNum = 0; uint32 pAlloc = 8388608; uint32 estID = ~uint32ZERO; bool *found = 0L; // From fixPolishesIID.c IIDdict = 0L; SEQdict = 0L; GENdict = 0L; // Incorporated from sortPolishes mergeFilesLen = 0; mergeFilesMax = sysconf(_SC_OPEN_MAX); mergeFiles = new FILE * [mergeFilesMax]; mergeNames = new char * [mergeFilesMax]; mergePolishes = new sim4polish * [mergeFilesMax]; // Default to printing stats on stdout. sFile = stdout; int arg = 1; while (arg < argc) { if (strcmp(argv[arg], "-n") == 0) { pAlloc = strtouint32(argv[++arg], 0L); } else if (strcmp(argv[arg], "-fpart") == 0) { arg++; fprintf(stderr, "reading query deflines from '%s'\n", argv[arg]); IIDdict = dict_create(DICTCOUNT_T_MAX, headerCompare); addToDict(IIDdict, argv[arg]); } else if (strcmp(argv[arg], "-g") == 0) { ++arg; fprintf(stderr, "reading genomic deflines from '%s'\n", argv[arg]); GENdict = dict_create(DICTCOUNT_T_MAX, headerCompare); addToDict(GENdict, argv[arg]); } else if (strcmp(argv[arg], "-F") == 0) { ++arg; fprintf(stderr, "reading query deflines from '%s'\n", argv[arg]); SEQdict = dict_create(DICTCOUNT_T_MAX, headerCompare); addToDict(SEQdict, argv[arg]); } else if (strcmp(argv[arg], "-f") == 0) { ++arg; SEQ = new seqCache(argv[arg]); } else if (strcmp(argv[arg], "-q") == 0) { ++arg; QLT = new seqCache(argv[arg]); } else if (strcmp(argv[arg], "-filter") == 0) { filter = atof(argv[++arg]); doFiltering = true; } else if (strcmp(argv[arg], "-output") == 0) { char cmd[1024] = {0}; errno = 0; ++arg; if (strcmp(argv[arg] + strlen(argv[arg]) - 4, ".bz2") == 0) { sprintf(cmd, "bzip2 -1c > %s", argv[arg]); oFile = popen(cmd, "w"); oFileIsPipe = 1; } else if (strcmp(argv[arg] + strlen(argv[arg]) - 3, ".gz") == 0) { sprintf(cmd, "gzip -1c > %s", argv[arg]); oFile = popen(cmd, "w"); oFileIsPipe = 1; } else { fprintf(stderr, "Got %s, not .bz2 not .gz!\n", argv[arg]); exit(1); } if (errno) fprintf(stderr, "Failed to open '%s': %s\n", cmd, strerror(errno)); doFiltering = true; } else if (strcmp(argv[arg], "-scores") == 0) { errno = 0; sFile = fopen(argv[++arg], "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", argv[arg-1], strerror(errno)); doFiltering = true; } else if (strcmp(argv[arg], "-unique") == 0) { char cmd[1024] = {0}; errno = 0; arg++; if (strcmp(argv[arg] + strlen(argv[arg]) - 4, ".bz2") == 0) sprintf(cmd, "bzip2 -1c > %s", argv[arg]); else if (strcmp(argv[arg] + strlen(argv[arg]) - 3, ".gz") == 0) sprintf(cmd, "gzip -1c > %s", argv[arg]); else sprintf(cmd, "cat > %s", argv[arg]); uFile = popen(cmd, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", cmd, strerror(errno)); doFiltering = true; } else if (strncmp(argv[arg], "-M", 2) == 0) { arg++; while ((arg < argc) && (fileExists(argv[arg]))) { if (mergeFilesLen >= mergeFilesMax) { fprintf(stderr, "%s: ERROR! Too many input files! Should be less than %d\n", argv[0], mergeFilesMax); exit(1); } mergeNames[mergeFilesLen] = argv[arg]; mergeFiles[mergeFilesLen++] = openFile(argv[arg], "r"); arg++; } arg--; } else { fprintf(stderr, "unknown option: %s\n", argv[arg]); } arg++; } if (doFiltering) { if (uFile == 0L) fprintf(stderr, "ERROR: -unique is required\n"), exit(1); if (sFile == 0L) fprintf(stderr, "ERROR: -scores is required\n"), exit(1); if ((filter < 0.0) || (filter > 1.0)) fprintf(stderr, "ERROR: -filter value of %f invalid. 0 <= F <= 100.\n", filter), exit(1); } if ((IIDdict == 0L) || (SEQdict == 0L) || (GENdict == 0L)) { fprintf(stderr, "WARNING! No sequence dictionaries, NOT FIXING IIDs! (supply -fpart, -f and -g)\n"); } if ((SEQ == 0L) || (QLT == 0L)) { fprintf(stderr, "I need -f and -q\n"); exit(1); } // We no longer require that input polishes be sorted increasingly; // now they only must be grouped. This remembers if we've seen a // match or not. At the end, we'll analyze() those we haven't done // already. // found = new bool [ SEQ->getNumberOfSequences() ]; for (uint32 i=0; i<SEQ->getNumberOfSequences(); i++) found[i] = false; // Initialize the merge -- if no merge files, nothing done! // for (int i=0; i<mergeFilesLen; i++) { mergePolishes[i] = new sim4polish(mergeFiles[i]); fixIID(mergePolishes[i], IIDdict); } // Read polishes, picking the best when we see a change in the // estID. sim4polish **p = new sim4polish * [pAlloc]; sim4polish *q; while ((q = nextPolish()) != 0L) { if ((q->_estID != estID) && (pNum > 0)) { //fprintf(stderr, "PickBest for estID "uint32FMT"\n", estID); found[estID] = true; pickBest(p, pNum); pNum = 0; } if (pNum >= pAlloc) { sim4polish **P = new sim4polish * [pAlloc * 2]; memcpy(p, P, sizeof(sim4polish *) * pAlloc); delete [] p; p = P; pAlloc *= 2; } p[pNum++] = q; estID = q->_estID; } if (pNum > 0) { found[estID] = true; pickBest(p, pNum); } // Attempt cleanup // for (int i=0; i<mergeFilesLen; i++) closeFile(mergeFiles[i], mergeNames[i]); for (estID=0; estID < SEQ->getNumberOfSequences(); estID++) if (found[estID] == false) analyze(estID, 0, SEQ->getSequenceLength(estID), SEQ->getSequenceLength(estID), true, 'M'); delete [] mergeFiles; delete [] mergeNames; delete [] mergePolishes; if (oFile) pclose(oFile); if (uFile) pclose(uFile); if (sFile) fclose(sFile); fprintf(stderr, "Uni:"uint32FMTW(8)" Con:"uint32FMTW(8)" (T:"uint32FMTW(8)" M:"uint32FMTW(8)" I:"uint32FMTW(8)" S:"uint32FMTW(8)" N:"uint32FMTW(8)") Inc:"uint32FMTW(8)" -- Save:"uint32FMTW(8)" Lost:"uint32FMTW(8)"\n", statOneMatch, statConsistent, consistentTie, consistentMatches, consistentIdentity, consistentTooShort, consistentNot, statInconsistent, statUnique, statLost); fprintf(stderr, "total: LQ:"uint32FMT" MQ:"uint32FMT" RQ:"uint32FMT"\n", totLQ, totMQ, totRQ); return(0); }
/// Convert all raw PAs in the group into a single set of "cooked" PAs /// under the leader. /// @param[in] ca the object pointer void ca_coalesce(ca_o ca) { dict_t *dict_raw, *dict_cooked; dnode_t *dnpr, *dnpc, *next; dict_raw = ca->ca_raw_pa_dict; assert(!ca->ca_cooked_pa_dict); if ((dict_cooked = dict_create(DICTCOUNT_T_MAX, pa_cmp_by_pathname))) { ca->ca_cooked_pa_dict = dict_cooked; } else { putil_syserr(2, "dict_create()"); } for (dnpr = dict_first(dict_raw); dnpr;) { pa_o raw_pa, ckd_pa; next = dict_next(dict_raw, dnpr); raw_pa = (pa_o)dnode_getkey(dnpr); _ca_verbosity_pa(raw_pa, ca, "COALESCING"); // All data is in the key - that's why the value can be null. if ((dnpc = dict_lookup(dict_cooked, raw_pa))) { int keep_cooked = 0; ckd_pa = (pa_o)dnode_getkey(dnpc); if (!pa_is_read(raw_pa) && !pa_is_read(ckd_pa)) { moment_s raw_timestamp, ckd_timestamp; // If they're both destructive ops (non-read) then we // need to consider timestamps and use the later one. if (pa_has_timestamp(raw_pa) && pa_has_timestamp(ckd_pa)) { // If the PA's have their own timestamps, use them. raw_timestamp = pa_get_timestamp(raw_pa); ckd_timestamp = pa_get_timestamp(ckd_pa); } else { // Otherwise key off the file times. This is for // support of "dummy" PAs as used in shopping. raw_timestamp = pa_get_moment(raw_pa); ckd_timestamp = pa_get_moment(ckd_pa); } if (moment_cmp(raw_timestamp, ckd_timestamp, NULL) <= 0) { // Cooked write op is newer and can stay. keep_cooked = 1; } } else if (pa_is_read(raw_pa)) { // There's no point replacing a read with another read, // so regardless of whether the current cooked PA is a // read or write, it can stay. keep_cooked = 1; } else { // A write always beats a read. } if (!keep_cooked) { dict_delete(dict_cooked, dnpc); dnode_destroy(dnpc); _ca_verbosity_pa(ckd_pa, ca, "REMOVING"); pa_destroy(ckd_pa); if (!(dnpc = dnode_create(NULL))) { putil_syserr(2, "dnode_create()"); } ckd_pa = pa_copy(raw_pa); dict_insert(dict_cooked, dnpc, ckd_pa); } } else { if (!(dnpc = dnode_create(NULL))) { putil_syserr(2, "dnode_create()"); } ckd_pa = pa_copy(raw_pa); dict_insert(dict_cooked, dnpc, ckd_pa); } // Clean up the raw set as we move PAs to the cooked one. dict_delete(dict_raw, dnpr); dnode_destroy(dnpr); dnpr = next; } return; }
/* Parses and executes the AGGREGATE procedure. */ int cmd_aggregate (struct lexer *lexer, struct dataset *ds) { struct dictionary *dict = dataset_dict (ds); struct agr_proc agr; struct file_handle *out_file = NULL; struct casereader *input = NULL, *group; struct casegrouper *grouper; struct casewriter *output = NULL; bool copy_documents = false; bool presorted = false; bool saw_direction; bool ok; memset(&agr, 0 , sizeof (agr)); agr.missing = ITEMWISE; agr.src_dict = dict; subcase_init_empty (&agr.sort); /* OUTFILE subcommand must be first. */ lex_match (lexer, T_SLASH); if (!lex_force_match_id (lexer, "OUTFILE")) goto error; lex_match (lexer, T_EQUALS); if (!lex_match (lexer, T_ASTERISK)) { out_file = fh_parse (lexer, FH_REF_FILE, dataset_session (ds)); if (out_file == NULL) goto error; } if (out_file == NULL && lex_match_id (lexer, "MODE")) { lex_match (lexer, T_EQUALS); if (lex_match_id (lexer, "ADDVARIABLES")) { agr.add_variables = true; /* presorted is assumed in ADDVARIABLES mode */ presorted = true; } else if (lex_match_id (lexer, "REPLACE")) { agr.add_variables = false; } else goto error; } if ( agr.add_variables ) agr.dict = dict_clone (dict); else agr.dict = dict_create (dict_get_encoding (dict)); dict_set_label (agr.dict, dict_get_label (dict)); dict_set_documents (agr.dict, dict_get_documents (dict)); /* Read most of the subcommands. */ for (;;) { lex_match (lexer, T_SLASH); if (lex_match_id (lexer, "MISSING")) { lex_match (lexer, T_EQUALS); if (!lex_match_id (lexer, "COLUMNWISE")) { lex_error_expecting (lexer, "COLUMNWISE", NULL); goto error; } agr.missing = COLUMNWISE; } else if (lex_match_id (lexer, "DOCUMENT")) copy_documents = true; else if (lex_match_id (lexer, "PRESORTED")) presorted = true; else if (lex_force_match_id (lexer, "BREAK")) { int i; lex_match (lexer, T_EQUALS); if (!parse_sort_criteria (lexer, dict, &agr.sort, &agr.break_vars, &saw_direction)) goto error; agr.break_var_cnt = subcase_get_n_fields (&agr.sort); if (! agr.add_variables) for (i = 0; i < agr.break_var_cnt; i++) dict_clone_var_assert (agr.dict, agr.break_vars[i]); /* BREAK must follow the options. */ break; } else goto error; } if (presorted && saw_direction) msg (SW, _("When PRESORTED is specified, specifying sorting directions " "with (A) or (D) has no effect. Output data will be sorted " "the same way as the input data.")); /* Read in the aggregate functions. */ lex_match (lexer, T_SLASH); if (!parse_aggregate_functions (lexer, dict, &agr)) goto error; /* Delete documents. */ if (!copy_documents) dict_clear_documents (agr.dict); /* Cancel SPLIT FILE. */ dict_set_split_vars (agr.dict, NULL, 0); /* Initialize. */ agr.case_cnt = 0; if (out_file == NULL) { /* The active dataset will be replaced by the aggregated data, so TEMPORARY is moot. */ proc_cancel_temporary_transformations (ds); proc_discard_output (ds); output = autopaging_writer_create (dict_get_proto (agr.dict)); } else { output = any_writer_open (out_file, agr.dict); if (output == NULL) goto error; } input = proc_open (ds); if (!subcase_is_empty (&agr.sort) && !presorted) { input = sort_execute (input, &agr.sort); subcase_clear (&agr.sort); } for (grouper = casegrouper_create_vars (input, agr.break_vars, agr.break_var_cnt); casegrouper_get_next_group (grouper, &group); casereader_destroy (group)) { struct casereader *placeholder = NULL; struct ccase *c = casereader_peek (group, 0); if (c == NULL) { casereader_destroy (group); continue; } initialize_aggregate_info (&agr); if ( agr.add_variables ) placeholder = casereader_clone (group); { struct ccase *cg; for (; (cg = casereader_read (group)) != NULL; case_unref (cg)) accumulate_aggregate_info (&agr, cg); } if (agr.add_variables) { struct ccase *cg; for (; (cg = casereader_read (placeholder)) != NULL; case_unref (cg)) dump_aggregate_info (&agr, output, cg); casereader_destroy (placeholder); } else { dump_aggregate_info (&agr, output, c); } case_unref (c); } if (!casegrouper_destroy (grouper)) goto error; if (!proc_commit (ds)) { input = NULL; goto error; } input = NULL; if (out_file == NULL) { struct casereader *next_input = casewriter_make_reader (output); if (next_input == NULL) goto error; dataset_set_dict (ds, agr.dict); dataset_set_source (ds, next_input); agr.dict = NULL; } else { ok = casewriter_destroy (output); output = NULL; if (!ok) goto error; } agr_destroy (&agr); fh_unref (out_file); return CMD_SUCCESS; error: if (input != NULL) proc_commit (ds); casewriter_destroy (output); agr_destroy (&agr); fh_unref (out_file); return CMD_CASCADING_FAILURE; }
static void mms_queue_run(char *dir, int (*deliver)(MmsEnvelope *), double sleepsecs, int num_threads, int *rstop) { int i, qstop = 0; List *stack = gwlist_create(); // static struct Qthread_t *tlist; struct Qthread_t *tlist; debug("", 0, "mms_queue_run: %s", dir); max_live_exceeded_warning_issued = 0; number_of_threads = num_threads; gw_assert(num_threads>0); if (tlists == NULL) tlists = dict_create(10, NULL); Octstr *odir = octstr_create(dir); tlist = gw_malloc(num_threads*sizeof tlist[0]); dict_put(tlists, odir, tlist); gw_assert(tlist == dict_get(tlists,odir)); debug("",0,"tlist allocated at addr %p", &tlist[0]); octstr_destroy(odir); for (i = 0; i<num_threads; i++) { /* Create threads for sending. */ debug("",0,"%s tlist[%d] has addr: %p", dir, i, &tlist[i]); tlist[i].l = gwlist_create(); gwlist_add_producer(tlist[i].l); tlist[i].deliver = deliver; gwthread_create((gwthread_func_t *)tdeliver, &tlist[i]); } i = 0; /* For stepping through above array. */ do { Octstr *xdir = NULL; gwlist_append(stack, octstr_create("")); /* Put initial dir on there. */ while (!*rstop && (xdir = gwlist_extract_first(stack)) != NULL) { int ret = run_dir(dir, octstr_get_cstr(xdir), tlist, num_threads, &i, stack); octstr_destroy(xdir); xdir = NULL; if (ret < 0) { if (ret <= -2) qstop = 1; goto qloop; } } octstr_destroy(xdir); if (*rstop) break; qloop: gwthread_sleep(sleepsecs); } while (!qstop); /* We are out of the queue, time to go away. */ for (i = 0; i<num_threads; i++) if (tlist[i].l) gwlist_remove_producer(tlist[i].l); gwthread_join_every((gwthread_func_t *)tdeliver); /* Wait for them all to terminate. */ for (i = 0; i<num_threads; i++) if (tlist[i].l) gwlist_destroy(tlist[i].l,NULL); /* Final destroy if needed. */ gw_free(tlist); gwlist_destroy(stack, (gwlist_item_destructor_t *)octstr_destroy); return; }
int main(int argc, char **args){ if(argc != 3){ fprintf(stderr,"%s RECV SEND\n",args[0]); exit(1); } signal(SIGINT,&call_for_stop); bstring pull_addr = bfromcstr(args[1]); bstring pub_addr = bfromcstr(args[2]); mongrel2_ctx *ctx = mongrel2_init(1); // Yes for threads? mongrel2_socket *pull_socket = mongrel2_pull_socket(ctx); mongrel2_connect(pull_socket, bdata(pull_addr)); pub_socket = mongrel2_pub_socket(ctx); mongrel2_connect(pub_socket, bdata(pub_addr)); mongrel2_set_identity(pub_socket, bdata(&SENDER) ); mongrel2_request *request; // Polling is done to show how to do a clean shutdown int poll_response; zmq_pollitem_t socket_tracker; socket_tracker.socket = pull_socket->zmq_socket; socket_tracker.events = ZMQ_POLLIN; // Let's try out some ADT goodness dict_t* dict = dict_create(DICTCOUNT_T_MAX, compare_session); dict_set_allocator(dict, alloc_dict, free_dict, NULL); dnode_t* tempnode = NULL; m2_ws_session_data *counter = NULL; int retval = 0; while(shutdown != 1){ poll_response = zmq_poll(&socket_tracker,1,500*1000); if(poll_response > 0){ request = mongrel2_recv(pull_socket); fprintf(stdout,"got something...\n"); if(request != NULL && mongrel2_request_for_disconnect(request) != 1){ m2_ws_session_id* incoming = calloc(1,sizeof(m2_ws_session_id)); incoming->req = request; printf("Looking at incoming->conn_id = %d\n",incoming->req->conn_id); tempnode = dict_lookup(dict,incoming); if(tempnode == NULL){ mongrel2_ws_reply_upgrade(request,pub_socket); counter = calloc(1,sizeof(m2_ws_session_data)); counter->times_seen = 0; retval = dict_alloc_insert(dict,incoming,counter); assert(retval == 1); } else { free(incoming); counter = dnode_get(tempnode); counter->times_seen += 1; } if(blength(request->body) > 0){ if(tempnode && mongrel2_ws_frame_get_fin(blength(request->body), (uint8_t*)bdata(request->body))){ printf("Hey, it's a close\n"); dict_delete_free(dict,tempnode); mongrel2_disconnect(pub_socket,request); } } else { bstring randmsg = genrandmsg(); mongrel2_ws_reply(pub_socket,request,randmsg); } printf("FYI: we've got %ld entries\n",dict_count(dict)); } else { fprintf(stdout,"Connection %d disconnected\n", request->conn_id); } } else if (poll_response < 0){ fprintf(stdout, "Error on poll!"); shutdown = 1; } } // bstring msg = bformat("{\"msg\" : \"hi there %d\"}", request->conn_id); // fprintf(stdout,"Sending new msg: '%*s'",blength(msg),bdata(msg)); // mongrel2_ws_reply(pub_socket,request,msg); // bdestroy(msg); // mongrel2_request_finalize(request); // mongrel2_reply(pub_socket,request,bfromcstr("")); bdestroy(pull_addr); bdestroy(pub_addr); mongrel2_close(pull_socket); mongrel2_close(pub_socket); mongrel2_deinit(ctx); fprintf(stdout,"\nClean shutdown done! Thanks for playing!\n"); return 0; }
int main() { size_t i = 0, m, max=100000; char buf[128]; struct timeval beg, end; long sec ; dict *d = dict_create(&opts); for(m = 0; m < 3; m++) { for(i = 0; i < max; i++) { memset(buf, 0, sizeof(buf)); snprintf(buf, sizeof(buf), ";;%lu;;", i); dict_add(d, buf, buf); } printf("add %d, %d %llu\n", DICT_USED(d), DICT_CAP(d),used_mem()); for(i = 0; i < max; i++) { dict_entry *entry; memset(buf, 0, sizeof(buf)); snprintf(buf, sizeof(buf), ";;%lu;;", i); entry = dict_find(d, buf); if(strcmp((char*)entry->value, (char*)buf) != 0) abort(); } printf("find %d, %d %llu\n", DICT_USED(d), DICT_CAP(d),used_mem()); for(i = 0; i < max; i++) { memset(buf, 0, sizeof(buf)); snprintf(buf, sizeof(buf), ";;%lu;;", i); dict_replace(d, buf, buf); } printf("prelace %d, %d %llu\n", DICT_USED(d), DICT_CAP(d),used_mem()); for(i = 0; i < max; i++) { memset(buf, 0, sizeof(buf)); snprintf(buf, sizeof(buf), ";;%lu;;", i); dict_del(d, buf); } printf("delete %d, %d %llu\n", DICT_USED(d), DICT_CAP(d),used_mem()); } dict_expand(d, max*2); printf("%d, %d %llu\n", DICT_USED(d), DICT_CAP(d),used_mem()); printf("%d %llu\n", DICT_CAP(d),used_mem()); dict_destroy(d); printf("%llu\n", used_mem()); d = dict_create(&opts); gettimeofday(&beg, NULL); for(i = 0; i < 1000000; i++) { memset(buf, 0, sizeof(buf)); snprintf(buf, sizeof(buf), "%lu", i); dict_add(d, buf, buf); } gettimeofday(&end, NULL); sec = (end.tv_sec - beg.tv_sec)*1000; sec += (end.tv_usec - beg.tv_usec)/1000; printf("add msec %ld, mm: %llu\n", sec, used_mem()); gettimeofday(&beg, NULL); dict_iterator *iter = dict_get_iterator(d); dict_entry *entry = NULL; i = 0; while((entry = dict_iterator_next(iter)) != NULL){ i++; } dict_iterator_destroy(iter); gettimeofday(&end, NULL); sec = (end.tv_sec - beg.tv_sec)*1000; sec += (end.tv_usec - beg.tv_usec)/1000; printf("iterator msec %ld, times: %lu mm: %llu\n", sec, i, used_mem()); gettimeofday(&beg, NULL); for(i = 0; i < 1000000; i++) { memset(buf, 0, sizeof(buf)); snprintf(buf, sizeof(buf), "%lu", i); dict_del(d, buf); } gettimeofday(&end, NULL); sec = (end.tv_sec - beg.tv_sec)*1000; sec += (end.tv_usec - beg.tv_usec)/1000; printf("remove msec %ld, mm: %llu\n", sec, used_mem()); dict_destroy(d); printf("%llu\r\n", used_mem()); return 0; }