Beispiel #1
0
END_TEST

START_TEST(test_dict_add3)
{
    TestData t, *pt;
    t.ivalue = 1;
    Dict *pdict = dict_create();
    dict_add(pdict, SIZEOF("key1"), (char*)&t, sizeof(TestData));

    pt = (TestData*)dict_get(pdict, SIZEOF("key1"));
    ck_assert_int_eq(pt->ivalue, 1);

    t.ivalue = 2;
    dict_add(pdict, SIZEOF("key1"), (char*)&t, sizeof(TestData));
    pt = (TestData*)dict_get(pdict, SIZEOF("key1"));
    ck_assert_int_eq(pt->ivalue, 1);
}
Beispiel #2
0
void _sql_init(void) {
  if (DBConnection < 0) {
    logging_register_module(sql);
    if (!_drivers) {
      _driver_mutex = mutex_create();
      _drivers = dict_create(NULL);
      dict_set_key_type(_drivers, &type_str);
      dict_set_data_type(_drivers, &type_int);
    }
    if (ErrorSQL < 1) {
      exception_register(ErrorSQL);
    }
    typedescr_register_with_methods(DBConnection, dbconn_t);
    typedescr_register(DBTransaction, tx_t);
  }
  assert(DBConnection);
}
Beispiel #3
0
void smpp_route_rebuild_database(SMPPServer *smpp_server) {
    info(0, "Rebuilding database routes");
    SMPPRouting *smpp_routing = smpp_server->routing;
    List *inbound_routes = smpp_database_get_routes(smpp_server, SMPP_ROUTE_DIRECTION_INBOUND, NULL); /* Only inbound are built, outbound are built when ESME's connect */
    
    List *old_inbound;
    Dict *old_outbound;
    
    gw_rwlock_wrlock(smpp_routing->lock);
    old_inbound = smpp_routing->inbound_routes;
    old_outbound = smpp_routing->outbound_routes;
    smpp_routing->inbound_routes = inbound_routes;
    smpp_routing->outbound_routes = dict_create(1024, (void(*)(void *))smpp_outbound_routes_destroy); /* Just reset, they will repopulate on their own */
    
    gw_rwlock_unlock(smpp_routing->lock);
    
    gwlist_destroy(old_inbound, (void(*)(void *))smpp_route_destroy);
    dict_destroy(old_outbound);
}
Beispiel #4
0
int cxml_node_addnode(CLOG_INFO* info, CXMLNODE *parent, CXMLNODE *child)
{
  int return_value=0;

  assert(NULL!=parent);
  assert(NULL!=child);

  // if the sub node is NULL create a new sub table
  if(NULL==parent->sub) {
    clog( info, CTRACE, "XML: xml_node_addnode(), creating sub node");
    parent->sub = dict_create(DICTCOUNT_T_MAX, (dict_comp_t)strcmp);
    // allow duplicates
    dict_allow_dupes(parent->sub);
  }

  // if the previous stuff worked, add the name/value pair
  if(NULL!=parent->sub && NULL!=child->name && NULL!=child->name->string) {

    dnode_t *node = (parent->sub)->dict_allocnode((parent->sub)->dict_context);
    char *key = strdup(child->name->string);

    clog( info, CTRACE, "XML: xml_node_adddata(), Saving sub node name=\"%.80s\"", child->name->string);

    if (node) {
      dnode_init(node, child);
      dict_insert(parent->sub, node, key);

      // let the child know who they are
      child->me = node;

      // success!
      return_value=1;
    }
  }

  // let the child know whow their parent is
  if(NULL!=parent && NULL!=child) {
    child->parent = parent;
  }

  return return_value;
}
static int tolua_dict_set_usertype(lua_State * L, int type)
{
    dict self = (dict)tolua_tousertype(L, 1, 0);
    const char *name = tolua_tostring(L, 2, 0);
    unit *value = tolua_tousertype(L, 3, 0);
    attrib *a = a_find(*self, &at_dict);
    variant val;

    val.v = value;

    for (; a && a->type == &at_dict; a = a->next) {
        if (strcmp(dict_name(a), name) == 0) {
            dict_set(a, type, val);
            return 0;
        }
    }

    a = a_add(self, dict_create(name, type, val));
    return 0;
}
int init_update(void)
{
    dict_types type;
    memset(&type, 0, sizeof(type));
    type.hash_function  = update_dict_hash_function;
    type.key_compare    = update_dict_key_compare;
    type.key_dup        = update_dict_key_dup;
    type.key_destructor = update_dict_key_free;
    type.val_dup        = update_dict_val_dup;
    type.val_destructor = update_dict_val_free;

    dict_update = dict_create(&type, 64);
    if (dict_update == NULL)
        return -__LINE__;

    nw_timer_set(&timer, 60, true, on_timer, NULL);
    nw_timer_start(&timer);

    return 0;
}
static int tolua_dict_set_number(lua_State * L)
{
    dict self = (dict)tolua_tousertype(L, 1, 0);
    const char *name = tolua_tostring(L, 2, 0);
    lua_Number value = tolua_tonumber(L, 3, 0);
    attrib *a = a_find(*self, &at_dict);
    variant val;

    val.f = (float)value;

    for (; a && a->type == &at_dict; a = a->next) {
        if (strcmp(dict_name(a), name) == 0) {
            dict_set(a, TREAL, val);
            return 0;
        }
    }

    a = a_add(self, dict_create(name, TREAL, val));
    return 0;
}
Beispiel #8
0
END_TEST

START_TEST(test_dict_add1)
{
    TestData t;
    t.ivalue = 1;
    Dict *pdict = dict_create();
    dict_add(pdict, SIZEOF("key1"), (char*)&t, sizeof(TestData));

    ck_assert_int_eq(dict_get_count(pdict), 1);

    TestData *t1 = (TestData*)dict_get(pdict, SIZEOF("key1"));
    ck_assert_msg(t1 != &t, "dict did not make a copy of input data");
    ck_assert_int_eq(t1->ivalue, 1);

    dict_add(pdict, SIZEOF("key2"), (char*)&t, sizeof(TestData));
    TestData *t2 = (TestData*)dict_get(pdict, SIZEOF("key2"));
    ck_assert_msg(t1 != t2, "cannot add the same data with two different keys");

    dict_free(pdict);
}
Beispiel #9
0
void testBasicDict() {
    struct dict *d = dict_create();

    dict_set(d, "name", "drmaa2");
    dict_del(d, "name");
    dict_set(d, "name", "drmaa2");
    assert(dict_has(d, "name") != 0);
    assert(dict_has(d, "language") == 0);

    const char *v = dict_get(d, "name");
    assert(strcmp(v, "drmaa2") == 0);

    dict_set(d, "language", "c");
    assert(dict_has(d, "language") != 0);

    dict_del(d, "name");
    assert(dict_has(d, "name") == 0);
    v = dict_get(d, "language");
    assert(strcmp(v, "c") == 0);

    dict_free(d);
}
Beispiel #10
0
void cmd_init()
{
	command_list = dict_create();
	dict_set_free_funcs(command_list, NULL, (dict_free_f *)cmd_free_subcmds);

	// Initialize our own commands
	cmd_register_list(commands, NULL);
	cmd_alias("exit", "quit", NULL);
	// Initialize external commands
	cmd_server_init();
	cmd_link_init();
	cmd_conf_init();
	cmd_oper_init();
	cmd_service_init();
	cmd_forward_init();
	cmd_feature_init();
	cmd_jupe_init();
	cmd_class_init();
	cmd_pseudo_init();
	cmd_client_init();
	cmd_webirc_init();
}
Beispiel #11
0
size_t get_file(const char *file_path, struct dict **dicts)
{
	char *token, buffer[1024];
	size_t size = 0, alloc = 1;
	struct dict *d = malloc(sizeof(struct dict));
	FILE *f = fopen(file_path, "r");
	while (fgets(buffer, sizeof(buffer), f) != NULL) {
		if ((token = strchr(buffer, '#')) != NULL)
			*token = '\0';
		if ((token = strchr(buffer, '=')) == NULL)
			continue;
		if (size == alloc) {
			alloc *= 2;
			d = realloc(d, alloc * sizeof(struct dict));
		}
		*token++ = '\0';
		while (*token == ' ' || *token == '\t')
			token++;
		dict_create(d + size++, buffer, strlen(buffer), token, strlen(token));
	}
	fclose(f);
	return size;
}
Beispiel #12
0
TEST(mydict, dict)
{
    dict_t* d = dict_create((void*)123);

    int k, v;
    dict_node_t* node = NULL;

    int i = 0;

    int sum = 0;
    for (i = 0; i < 10000; i++) {
        k = i, v = i;
        node = dict_add(d, &k, sizeof(k), &v, sizeof(v));
        ASSERT_TRUE(node != NULL);

        sum += i;
    }
    for (i = 0; i < 10000; i++) {
        node = dict_find(d, &k, sizeof(k));
        ASSERT_EQ((long)dict_get_val(node), i);
    }

    i = 0;
    for(node = dict_first(d); node; node = dict_next(d, node)) {
        i += (long)dict_get_val(node);
    }
    ASSERT_EQ(i, sum);

    for (i = 0; i < 10000; i++) {
        k = i;
        node = dict_delete(d, &k, sizeof(k));
        ASSERT_EQ((long)dict_get_val(node), i);
    }
    ASSERT_TRUE(dict_empty(d));

    dict_destroy(d);
}
Beispiel #13
0
int store_file_init(const Octstr *fname, long dump_freq)
{
    /* Initialize function pointers */
    store_messages = store_file_messages;
    store_save = store_file_save;
    store_save_ack = store_file_save_ack;
    store_load = store_file_load;
    store_dump = store_file_dump;
    store_shutdown = store_file_shutdown;
    store_status = store_file_status;

    if (fname == NULL)
        return 0; /* we are done */

    if (octstr_len(fname) > (FILENAME_MAX-5))
        panic(0, "Store file filename too long: `%s', failed to init.",
	      octstr_get_cstr(fname));

    filename = octstr_duplicate(fname);
    newfile = octstr_format("%s.new", octstr_get_cstr(filename));
    bakfile = octstr_format("%s.bak", octstr_get_cstr(filename));

    sms_dict = dict_create(1024, msg_destroy_item);

    if (dump_freq > 0)
        dump_frequency = dump_freq;
    else
        dump_frequency = BB_STORE_DEFAULT_DUMP_FREQ;

    file_mutex = mutex_create();
    active = 1;

    loaded = gwlist_create();
    gwlist_add_producer(loaded);

    return 0;
}
Beispiel #14
0
/* Set the clip according to the currently
   selected range in the data sheet */
static void
data_sheet_set_clip (PsppireSheet *sheet)
{
  int i;
  struct casewriter *writer ;
  PsppireSheetRange range;
  PsppireDataStore *ds;
  struct case_map *map = NULL;
  casenumber max_rows;
  size_t max_columns;
  gint row0, rowi;
  gint col0, coli;

  ds = PSPPIRE_DATA_STORE (psppire_sheet_get_model (sheet));

  psppire_sheet_get_selected_range (sheet, &range);

  col0 = MIN (range.col0, range.coli);
  coli = MAX (range.col0, range.coli);
  row0 = MIN (range.row0, range.rowi);
  rowi = MAX (range.row0, range.rowi);

   /* If nothing selected, then use active cell */
  if ( row0 < 0 || col0 < 0 )
    {
      gint row, col;
      psppire_sheet_get_active_cell (sheet, &row, &col);

      row0 = rowi = row;
      col0 = coli = col;
    }

  /* The sheet range can include cells that do not include data.
     Exclude them from the range. */
  max_rows = psppire_data_store_get_case_count (ds);
  if (rowi >= max_rows)
    {
      if (max_rows == 0)
        return;
      rowi = max_rows - 1;
    }
  max_columns = dict_get_var_cnt (ds->dict->dict);
  if (coli >= max_columns)
    {
      if (max_columns == 0)
        return;
      coli = max_columns - 1;
    }

  /* Destroy any existing clip */
  if ( clip_datasheet )
    {
      casereader_destroy (clip_datasheet);
      clip_datasheet = NULL;
    }

  if ( clip_dict )
    {
      dict_destroy (clip_dict);
      clip_dict = NULL;
    }

  /* Construct clip dictionary. */
  clip_dict = dict_create (dict_get_encoding (ds->dict->dict));
  for (i = col0; i <= coli; i++)
    dict_clone_var_assert (clip_dict, dict_get_var (ds->dict->dict, i));

  /* Construct clip data. */
  map = case_map_by_name (ds->dict->dict, clip_dict);
  writer = autopaging_writer_create (dict_get_proto (clip_dict));
  for (i = row0; i <= rowi ; ++i )
    {
      struct ccase *old = psppire_data_store_get_case (ds, i);
      if (old != NULL)
        casewriter_write (writer, case_map_execute (map, old));
      else
        casewriter_force_error (writer);
    }
  case_map_destroy (map);

  clip_datasheet = casewriter_make_reader (writer);

  data_sheet_update_clipboard (sheet);
}
Beispiel #15
0
int main(int argc,char *argv[],char *env[])
{
	pdict = dict_create();

	FILE *fp = fopen(FILE_NAME,"r");
	if(fp == NULL)
	{
		perror("fopen");
		return FAILED;
	}
	
	char english[1000];
	char chinese[1000];

	while(!feof(fp))
	{
		memset(english,0,sizeof(english));
		memset(chinese,0,sizeof(chinese));
		
		if(fgets(english,sizeof(english),fp) == NULL)
			break;
		*strchr(english,'\n') = 0;
		
		if(fgets(chinese,sizeof(chinese),fp) == NULL)
			break;
		*strchr(chinese,'\n') = 0;

		dict_insert(pdict,english,chinese);
	}

	signal(28,handle);

	show_window();
	move_xy(3,15);

	char key;
	while(1)
	{
		key = get_key();	
		switch(key)
		{
			case ESC:
				clear_screen();
				move_xy(1,1);
				return 0;
			case BACKSPACE:
				if(buf_i > 0)
				{
					move_left(1);
					putchar(' ');
					fflush(stdout);
					move_left(1);

					buf_i--;
					input_buf[buf_i] = 0;

					show_word(pdict,input_buf);
				}
				break;
			default:
			if(buf_i <= win_y - 20 && isprint(key))
			{
				putchar(key);
				fflush(stdout);

				input_buf[buf_i] = key;
				buf_i++;
				input_buf[buf_i] = 0;

				show_word(pdict,input_buf);
			}	
		}
	}

	return 0;
}
Beispiel #16
0
static void run_smsbox(void *arg)
{
    Boxc *newconn;
    long sender;
    Msg *msg;
    List *keys;
    Octstr *key;

    gwlist_add_producer(flow_threads);
    newconn = arg;
    newconn->incoming = gwlist_create();
    gwlist_add_producer(newconn->incoming);
    newconn->retry = incoming_sms;
    newconn->outgoing = outgoing_sms;
    newconn->sent = dict_create(smsbox_max_pending, NULL);
    newconn->pending = semaphore_create(smsbox_max_pending);

    sender = gwthread_create(boxc_sender, newconn);
    if (sender == -1) {
        error(0, "Failed to start a new thread, disconnecting client <%s>",
              octstr_get_cstr(newconn->client_ip));
        goto cleanup;
    }
    /*
     * We register newconn in the smsbox_list here but mark newconn as routable
     * after identification or first message received from smsbox. So we can avoid
     * a race condition for routable smsboxes (otherwise between startup and
     * registration we will forward some messages to smsbox).
     */
    gw_rwlock_wrlock(smsbox_list_rwlock);
    gwlist_append(smsbox_list, newconn);
    gw_rwlock_unlock(smsbox_list_rwlock);

    gwlist_add_producer(newconn->outgoing);
    boxc_receiver(newconn);
    gwlist_remove_producer(newconn->outgoing);

    /* remove us from smsbox routing list */
    gw_rwlock_wrlock(smsbox_list_rwlock);
    gwlist_delete_equal(smsbox_list, newconn);
    if (newconn->boxc_id) {
        dict_remove(smsbox_by_id, newconn->boxc_id);
    }
    gw_rwlock_unlock(smsbox_list_rwlock);

    /*
     * check if we in the shutdown phase and sms dequeueing thread
     *   has removed the producer already
     */
    if (gwlist_producer_count(newconn->incoming) > 0)
        gwlist_remove_producer(newconn->incoming);

    /* check if we are still waiting for ack's and semaphore locked */
    if (dict_key_count(newconn->sent) >= smsbox_max_pending)
        semaphore_up(newconn->pending); /* allow sender to go down */
        
    gwthread_join(sender);

    /* put not acked msgs into incoming queue */    
    keys = dict_keys(newconn->sent);
    while((key = gwlist_extract_first(keys)) != NULL) {
        msg = dict_remove(newconn->sent, key);
        gwlist_produce(incoming_sms, msg);
        octstr_destroy(key);
    }
    gw_assert(gwlist_len(keys) == 0);
    gwlist_destroy(keys, octstr_destroy_item);

    /* clear our send queue */
    while((msg = gwlist_extract_first(newconn->incoming)) != NULL) {
        gwlist_produce(incoming_sms, msg);
    }

cleanup:
    gw_assert(gwlist_len(newconn->incoming) == 0);
    gwlist_destroy(newconn->incoming, NULL);
    gw_assert(dict_key_count(newconn->sent) == 0);
    dict_destroy(newconn->sent);
    semaphore_destroy(newconn->pending);
    boxc_destroy(newconn);

    /* wakeup the dequeueing thread */
    gwthread_wakeup(sms_dequeue_thread);

    gwlist_remove_producer(flow_threads);
}
Beispiel #17
0
int smpp_pdu_init(Cfg *cfg)
{
    CfgGroup *grp;
    List *l;

    if (initialized)
        return 0;

    l = cfg_get_multi_group(cfg, octstr_imm("smpp-tlv"));
    tlvs = gwlist_create();
    tlvs_by_tag = dict_create(1024, (void(*)(void*))dict_destroy);
    tlvs_by_name = dict_create(1024, (void(*)(void*))dict_destroy);
    while (l != NULL && (grp = gwlist_extract_first(l)) != NULL) {
        struct smpp_tlv *tlv;
        Octstr *tmp, *smsc_id;
        List *l2;

        tlv = gw_malloc(sizeof(*tlv));
        if ((tlv->name = cfg_get(grp, octstr_imm("name"))) == NULL) {
            error(0, "SMPP: Unable to get name for smpp-tlv.");
            smpp_tlv_destroy(tlv);
            goto failed;
        }
        if (cfg_get_integer(&tlv->tag, grp, octstr_imm("tag")) == -1) {
            error(0, "SMPP: Unable to get tag for smpp-tlv.");
            smpp_tlv_destroy(tlv);
            goto failed;
        }
        if (cfg_get_integer(&tlv->length, grp, octstr_imm("length")) == -1) {
            error(0, "SMPP: Unable to get length for smpp-tlv.");
            smpp_tlv_destroy(tlv);
            goto failed;
        }
        if ((tmp = cfg_get(grp, octstr_imm("type"))) == NULL) {
            error(0, "SMPP: Unable to get type for smpp-tlv.");
            smpp_tlv_destroy(tlv);
            goto failed;
        }
        if (octstr_str_case_compare(tmp, "octetstring") == 0)
            tlv->type = SMPP_TLV_OCTETS;
        else if (octstr_str_case_compare(tmp, "nulterminated") == 0)
            tlv->type = SMPP_TLV_NULTERMINATED;
        else if (octstr_str_case_compare(tmp, "integer") == 0)
            tlv->type = SMPP_TLV_INTEGER;
        else {
            error(0, "SMPP: Unknown type for smpp-tlv: `%s'", octstr_get_cstr(tmp));
            octstr_destroy(tmp);
            smpp_tlv_destroy(tlv);
            goto failed;
        }
        octstr_destroy(tmp);

        /* put to all TLVs */
        gwlist_produce(tlvs, tlv);

        smsc_id = cfg_get(grp, octstr_imm("smsc-id"));
        if (smsc_id != NULL) {
            l2 = octstr_split(smsc_id, octstr_imm(";"));
            octstr_destroy(smsc_id);
        } else {
            l2 = gwlist_create();
            gwlist_produce(l2, octstr_create(DEFAULT_SMSC_ID));
        }
        while(l2 != NULL && (smsc_id = gwlist_extract_first(l2)) != NULL) {
            Dict *tmp_dict;

            debug("sms.smpp", 0, "adding smpp-tlv for smsc-id=%s", octstr_get_cstr(smsc_id));

            tmp_dict = dict_get(tlvs_by_name, smsc_id);
            if (tmp_dict == NULL) {
                tmp_dict = dict_create(1024, NULL);
                dict_put(tlvs_by_name, smsc_id, tmp_dict);
            }
            /* put into dict */
            if (!dict_put_once(tmp_dict, tlv->name, tlv)) {
                error(0, "SMPP: Double TLV name %s found.", octstr_get_cstr(tlv->name));
                octstr_destroy(smsc_id);
                goto failed;
            }

            tmp_dict = dict_get(tlvs_by_tag, smsc_id);
            if (tmp_dict == NULL) {
                tmp_dict = dict_create(1024, NULL);
                dict_put(tlvs_by_tag, smsc_id, tmp_dict);
            }
            tmp = octstr_format("%ld", tlv->tag);
            if (!dict_put_once(tmp_dict, tmp, tlv)) {
                error(0, "SMPP: Double TLV tag %s found.", octstr_get_cstr(tmp));
                gwlist_destroy(l2, octstr_destroy_item);
                octstr_destroy(tmp);
                octstr_destroy(smsc_id);
                goto failed;
            }
            octstr_destroy(tmp);
            octstr_destroy(smsc_id);
        }
        gwlist_destroy(l2, octstr_destroy_item);
    }
    gwlist_destroy(l, NULL);

    initialized = 1;
    return 0;

failed:
    gwlist_destroy(tlvs, (void(*)(void*))smpp_tlv_destroy);
    dict_destroy(tlvs_by_tag);
    dict_destroy(tlvs_by_name);
    return -1;
}
Beispiel #18
0
mCfg *mms_cfg_read(Octstr *file)
{
     Octstr *sf;
     List *lines;
     int i, n;
     mCfg *cfg;
     mCfgGrp *grp = NULL;
     int skip = 0;
     
     gw_assert(file);

     if ((sf = octstr_read_file(octstr_get_cstr(file))) == NULL) {
	  mms_error(errno, "mms_cfg", NULL, "failed to read config from `%s'", octstr_get_cstr(file));
	  return NULL;
     }

     cfg = gw_malloc(sizeof *cfg);
     cfg->file = octstr_duplicate(file);
     cfg->grps = dict_create(7, NULL);

     cfg->xcfg = NULL;
     cfg->cfg_funcs = NULL;
     
     lines = octstr_split(sf, octstr_imm("\n"));    
     for (i = 0, n = gwlist_len(lines); i < n; i++) {
	  Octstr *current = gwlist_get(lines,i);
	  int pos;
	  
	  octstr_strip_blanks(current);
	  
	  if (octstr_len(current) == 0) { /* end of group. */
	       grp = NULL;
	       skip = 0;
	       continue;
	  } else if (skip || octstr_get_char(current, 0) == '#') 
	       continue; 
	  	  
	  if ((pos = octstr_search_char(current, '=',0)) > 0) {
	       /* a field name. first see if start of grp */
	       Octstr *field = octstr_copy(current,0,pos);
	       Octstr *value = octstr_copy(current,pos+1,octstr_len(current));
	       
	       octstr_strip_blanks(field);
	       fixup_value(value, i+1);
#if 0
	       mms_info(0, "mms_cfg", NULL, "field/value: [%s - %s]", octstr_get_cstr(field), 
		    octstr_get_cstr(value));
#endif

	       if (octstr_str_case_compare(field, "group") == 0) 
		    if (grp == NULL) { /* grp name. */		    
			 int ismulti = is_multigroup(value);
			 
			 if (ismulti < 0) {
			      mms_info(0, "mms_cfg", NULL, "Skipping unknown group `%s' at line %d of conf file", 
				   octstr_get_cstr(value), i+1);
			      skip = 1;
			 } else {
			      grp = gw_malloc(sizeof *grp);
			      grp->name = octstr_duplicate(value);
			      grp->fields = dict_create(23, (void (*)(void *))octstr_destroy);
			      
			      if (ismulti) {
				   List *l = dict_get(cfg->grps, value);
				   
				   if (l == NULL) { 
					l = gwlist_create();
					dict_put(cfg->grps, value, l);			      
				   }
				   gwlist_append(l, grp);			 
			      } else if (dict_put_once(cfg->grps, value, grp) == 0)
				   panic(0, "Group `%s' [at line %d] cannot appear more "
					 "than once in config!",
					 octstr_get_cstr(value), i+1);
			 }
		    } else
			 panic(0,"`group' is an illegal field name "
			       "within a group at line %d in config file!",
			       i+1);
	       else  if (grp) /* an ordinary field name. */
		    check_and_add_field(grp, field, value,i+1);
	       else 
		    panic(0, "A group must begin with a `group = group_name' "
			  "clause [at line %d in config file]", i+1);			      	       
	       
	       octstr_destroy(field);
	       octstr_destroy(value);
	  } else
	       panic(0, "mal-formed entry in conf file at line %d!", i+1);
     }

     gwlist_destroy(lines, (gwlist_item_destructor_t *)octstr_destroy);
     octstr_destroy(sf);

     /* Now check if config-source is set, use that. */
     if ((grp = mms_cfg_get_single(cfg, octstr_imm("config-source"))) != NULL) {
	  Octstr *init = mms_cfg_get(cfg, grp, octstr_imm("config-library-init-param"));
	  cfg->cfg_funcs = _mms_load_module(cfg, grp, "config-library", "cfg_funcs", NULL);
	  
	  if (cfg->cfg_funcs == NULL ||
	      cfg->cfg_funcs->read == NULL ||
	      (cfg->xcfg = cfg->cfg_funcs->read(init)) == NULL) {
	       mms_error(0, "mms_cfg", NULL, "Failed to load cfg reader library from conf!");
	       mms_cfg_destroy(cfg);
	       cfg = NULL;
	  }
	  
	  octstr_destroy(init);	  
     }
     
     return cfg;
}
Beispiel #19
0
static int
combine_files (enum comb_command_type command,
               struct lexer *lexer, struct dataset *ds)
{
  struct comb_proc proc;

  bool saw_by = false;
  bool saw_sort = false;
  struct casereader *active_file = NULL;

  char *first_name = NULL;
  char *last_name = NULL;

  struct taint *taint = NULL;

  size_t n_tables = 0;
  size_t allocated_files = 0;

  size_t i;

  proc.files = NULL;
  proc.n_files = 0;
  proc.dict = dict_create (get_default_encoding ());
  proc.output = NULL;
  proc.matcher = NULL;
  subcase_init_empty (&proc.by_vars);
  proc.first = NULL;
  proc.last = NULL;
  proc.buffered_case = NULL;
  proc.prev_BY = NULL;

  dict_set_case_limit (proc.dict, dict_get_case_limit (dataset_dict (ds)));

  lex_match (lexer, T_SLASH);
  for (;;)
    {
      struct comb_file *file;
      enum comb_file_type type;

      if (lex_match_id (lexer, "FILE"))
        type = COMB_FILE;
      else if (command == COMB_MATCH && lex_match_id (lexer, "TABLE"))
        {
          type = COMB_TABLE;
          n_tables++;
        }
      else
        break;
      lex_match (lexer, T_EQUALS);

      if (proc.n_files >= allocated_files)
        proc.files = x2nrealloc (proc.files, &allocated_files,
                                sizeof *proc.files);
      file = &proc.files[proc.n_files++];
      file->type = type;
      subcase_init_empty (&file->by_vars);
      subcase_init_empty (&file->src);
      subcase_init_empty (&file->dst);
      file->mv = NULL;
      file->handle = NULL;
      file->dict = NULL;
      file->reader = NULL;
      file->data = NULL;
      file->is_sorted = true;
      file->in_name = NULL;
      file->in_var = NULL;

      if (lex_match (lexer, T_ASTERISK))
        {
          if (!dataset_has_source (ds))
            {
              msg (SE, _("Cannot specify the active dataset since none "
                         "has been defined."));
              goto error;
            }

          if (proc_make_temporary_transformations_permanent (ds))
            msg (SE, _("This command may not be used after TEMPORARY when "
                       "the active dataset is an input source.  "
                       "Temporary transformations will be made permanent."));

          file->dict = dict_clone (dataset_dict (ds));
        }
      else
        {
          file->handle = fh_parse (lexer, FH_REF_FILE, dataset_session (ds));
          if (file->handle == NULL)
            goto error;

          file->reader = any_reader_open (file->handle, NULL, &file->dict);
          if (file->reader == NULL)
            goto error;
        }

      while (lex_match (lexer, T_SLASH))
        if (lex_match_id (lexer, "RENAME"))
          {
            if (!parse_dict_rename (lexer, file->dict))
              goto error;
          }
        else if (lex_match_id (lexer, "IN"))
          {
            lex_match (lexer, T_EQUALS);
            if (lex_token (lexer) != T_ID)
              {
                lex_error (lexer, NULL);
                goto error;
              }

            if (file->in_name)
              {
                msg (SE, _("Multiple IN subcommands for a single FILE or "
                           "TABLE."));
                goto error;
              }
            file->in_name = xstrdup (lex_tokcstr (lexer));
            lex_get (lexer);
          }
        else if (lex_match_id (lexer, "SORT"))
          {
            file->is_sorted = false;
            saw_sort = true;
          }

      if (!merge_dictionary (proc.dict, file))
        goto error;
    }

  while (lex_token (lexer) != T_ENDCMD)
    {
      if (lex_match (lexer, T_BY))
	{
          const struct variable **by_vars;
          size_t i;
          bool ok;

	  if (saw_by)
	    {
              lex_sbc_only_once ("BY");
	      goto error;
	    }
          saw_by = true;

	  lex_match (lexer, T_EQUALS);
          if (!parse_sort_criteria (lexer, proc.dict, &proc.by_vars,
                                    &by_vars, NULL))
	    goto error;

          ok = true;
          for (i = 0; i < proc.n_files; i++)
            {
              struct comb_file *file = &proc.files[i];
              size_t j;

              for (j = 0; j < subcase_get_n_fields (&proc.by_vars); j++)
                {
                  const char *name = var_get_name (by_vars[j]);
                  struct variable *var = dict_lookup_var (file->dict, name);
                  if (var != NULL)
                    subcase_add_var (&file->by_vars, var,
                                     subcase_get_direction (&proc.by_vars, j));
                  else
                    {
                      if (file->handle != NULL)
                        msg (SE, _("File %s lacks BY variable %s."),
                             fh_get_name (file->handle), name);
                      else
                        msg (SE, _("Active dataset lacks BY variable %s."),
                             name);
                      ok = false;
                    }
                }
              assert (!ok || subcase_conformable (&file->by_vars,
                                                  &proc.files[0].by_vars));
            }
          free (by_vars);

          if (!ok)
            goto error;
	}
      else if (command != COMB_UPDATE && lex_match_id (lexer, "FIRST"))
        {
          if (first_name != NULL)
            {
              lex_sbc_only_once ("FIRST");
              goto error;
            }

	  lex_match (lexer, T_EQUALS);
          if (!lex_force_id (lexer))
            goto error;
          first_name = xstrdup (lex_tokcstr (lexer));
          lex_get (lexer);
        }
      else if (command != COMB_UPDATE && lex_match_id (lexer, "LAST"))
        {
          if (last_name != NULL)
            {
              lex_sbc_only_once ("LAST");
              goto error;
            }

	  lex_match (lexer, T_EQUALS);
          if (!lex_force_id (lexer))
            goto error;
          last_name = xstrdup (lex_tokcstr (lexer));
          lex_get (lexer);
        }
      else if (lex_match_id (lexer, "MAP"))
	{
	  /* FIXME. */
	}
      else if (lex_match_id (lexer, "DROP"))
        {
          if (!parse_dict_drop (lexer, proc.dict))
            goto error;
        }
      else if (lex_match_id (lexer, "KEEP"))
        {
          if (!parse_dict_keep (lexer, proc.dict))
            goto error;
        }
      else
	{
	  lex_error (lexer, NULL);
	  goto error;
	}

      if (!lex_match (lexer, T_SLASH) && lex_token (lexer) != T_ENDCMD)
        {
          lex_end_of_command (lexer);
          goto error;
        }
    }

  if (!saw_by)
    {
      if (command == COMB_UPDATE)
        {
          lex_sbc_missing ("BY");
          goto error;
        }
      if (n_tables)
        {
          msg (SE, _("BY is required when %s is specified."), "TABLE");
          goto error;
        }
      if (saw_sort)
        {
          msg (SE, _("BY is required when %s is specified."), "SORT");
          goto error;
        }
    }

  /* Add IN, FIRST, and LAST variables to master dictionary. */
  for (i = 0; i < proc.n_files; i++)
    {
      struct comb_file *file = &proc.files[i];
      if (!create_flag_var ("IN", file->in_name, proc.dict, &file->in_var))
        goto error;
    }
  if (!create_flag_var ("FIRST", first_name, proc.dict, &proc.first)
      || !create_flag_var ("LAST", last_name, proc.dict, &proc.last))
    goto error;

  dict_delete_scratch_vars (proc.dict);
  dict_compact_values (proc.dict);

  /* Set up mapping from each file's variables to master
     variables. */
  for (i = 0; i < proc.n_files; i++)
    {
      struct comb_file *file = &proc.files[i];
      size_t src_var_cnt = dict_get_var_cnt (file->dict);
      size_t j;

      file->mv = xnmalloc (src_var_cnt, sizeof *file->mv);
      for (j = 0; j < src_var_cnt; j++)
        {
          struct variable *src_var = dict_get_var (file->dict, j);
          struct variable *dst_var = dict_lookup_var (proc.dict,
                                                      var_get_name (src_var));
          if (dst_var != NULL)
            {
              size_t n = subcase_get_n_fields (&file->src);
              file->mv[n] = var_get_missing_values (src_var);
              subcase_add_var (&file->src, src_var, SC_ASCEND);
              subcase_add_var (&file->dst, dst_var, SC_ASCEND);
            }
        }
    }

  proc.output = autopaging_writer_create (dict_get_proto (proc.dict));
  taint = taint_clone (casewriter_get_taint (proc.output));

  /* Set up case matcher. */
  proc.matcher = case_matcher_create ();
  for (i = 0; i < proc.n_files; i++)
    {
      struct comb_file *file = &proc.files[i];
      if (file->reader == NULL)
        {
          if (active_file == NULL)
            {
              proc_discard_output (ds);
              file->reader = active_file = proc_open_filtering (ds, false);
            }
          else
            file->reader = casereader_clone (active_file);
        }
      if (!file->is_sorted)
        file->reader = sort_execute (file->reader, &file->by_vars);
      taint_propagate (casereader_get_taint (file->reader), taint);
      file->data = casereader_read (file->reader);
      if (file->type == COMB_FILE)
        case_matcher_add_input (proc.matcher, &file->by_vars,
                                &file->data, &file->is_minimal);
    }

  if (command == COMB_ADD)
    execute_add_files (&proc);
  else if (command == COMB_MATCH)
    execute_match_files (&proc);
  else if (command == COMB_UPDATE)
    execute_update (&proc);
  else
    NOT_REACHED ();

  case_matcher_destroy (proc.matcher);
  proc.matcher = NULL;
  close_all_comb_files (&proc);
  if (active_file != NULL)
    proc_commit (ds);

  dataset_set_dict (ds, proc.dict);
  dataset_set_source (ds, casewriter_make_reader (proc.output));
  proc.dict = NULL;
  proc.output = NULL;

  free_comb_proc (&proc);

  free (first_name);
  free (last_name);

  return taint_destroy (taint) ? CMD_SUCCESS : CMD_CASCADING_FAILURE;

 error:
  if (active_file != NULL)
    proc_commit (ds);
  free_comb_proc (&proc);
  taint_destroy (taint);
  free (first_name);
  free (last_name);
  return CMD_CASCADING_FAILURE;
}
/*************************************************************************
*       dict_import: read in an ASCII dictionary.
*
*       dict_fname - name of dictionary file
*       parameters to create a DICTIONARY structure (see dict_create)
*
*       Returns: pointer to created DICTIONARY structure
*                (NULL on failure)
*
*************************************************************************/
DICTIONARY *dict_import( const char *dict_fname ,
                         const long initial_string_count ,
                         const long initial_hash_entries ,
                         const long max_chain_length )
{
        DICTIONARY     *dict;
        char           buffer[BUFLEN], ch;
        int            index, c, c0;
        long           number;
        FILE           *fi = NULL;

        /***********
        **  Dictionary setup.
        ***********/

        dict = dict_create( 4,
                            initial_string_count ,
                            initial_hash_entries ,
                            max_chain_length );
        if ( dict == NULL )
           goto err_exit;

        /***********
        **  Read the dictionary file
        **  Each line should have one word or a string delimited by '|'
        ***********/

        if ( (fi=fopen(dict_fname,"r")) == NULL )
                goto err_exit;
        while( fgets(buffer,BUFLEN,fi) != NULL ) {
                c0 = 0;
                   /*  Skip to non-blank  */
                while ( (c0<BUFLEN-2) && (buffer[c0]==' ') ) ++c0;
                if ( buffer[c0] == '|' ) {
                        c = ++c0;
                        ch = '|';
                } else {
                        c = c0;
                        ch = ' ';
                } /* endif */
                   /*  Scan to blank or matching '|' */
                while ( (c<BUFLEN-1) && (buffer[c]!='\0') &&
                        (buffer[c]!='\n') && (buffer[c]!=ch)  )
                        ++c;
                buffer[c] = '\0';
                   /*  Insert the word  */
                if ( dict_insert(dict,buffer+c0,1,0,NULL,&number) == NULL )
                        goto err_exit;
        } /* endwhile */

        /***********
        **  Fill in the dictionary parameter vector.
        ***********/

        if ( dict_set_parm_values(dict) == FALSE )
                goto err_exit;

        /***********
        **  Update the table of contents for HASH STTB STAR
        ***********/

        if ( (index=dict_toc_index(dict,"HASH")) == -1 )
                goto err_exit;
        dict->toc[index].size = dict->table_size * sizeof(long);

        if ( (index=dict_toc_index(dict,"STTB")) == -1 )
                goto err_exit;
        dict->toc[index].size = dict->string_max * sizeof(char);

        if ( (index=dict_toc_index(dict,"STAR")) == -1 )
                goto err_exit;
        dict->toc[index].size = dict->array_size * sizeof(STRING_ENTRY);

           /*  Success. Return a pointer to the new dictionary.  */
        fclose(fi);
        return( dict );

           /*  Failure. Ignominiously erase our tracks and return NULL.  */
err_exit:
        if ( fi != NULL )
           fclose(fi);
        dict_destroy( dict );
        return NULL;
}
Beispiel #21
0
dict_t *
minerva_dict_create(void)
{
    return dict_create();
}
Beispiel #22
0
int
main(int argc, char **argv) {

  char   *merylCount = 0L;
  char   *fastaName = 0L;

  int arg=1;
  while (arg < argc) {

    if        (strcmp(argv[arg], "-m") == 0) {
      merylCount = argv[++arg];
    } else if (strcmp(argv[arg], "-f") == 0) {
      fastaName = argv[++arg];
    } else {
      fprintf(stderr, "unknown option '%s'\n", argv[arg]);
    }
    arg++;
  }

  if ((merylCount == 0L) || (fastaName == 0L)) {
    fprintf(stderr, "usage: %s -m <meryl-name-prefix> -f <fasta-file>\n", argv[0]);
    exit(1);
  }


  //  Open the count files
  //
  merylStreamReader  *MSR = new merylStreamReader(merylCount);

  fprintf(stderr, "Mers are "uint32FMT" bases.\n", MSR->merSize());
  fprintf(stderr, "There are "uint64FMT" unique (copy = 1) mers.\n", MSR->numberOfUniqueMers());
  fprintf(stderr, "There are "uint64FMT" distinct mers.\n", MSR->numberOfDistinctMers());
  fprintf(stderr, "There are "uint64FMT" mers total.\n", MSR->numberOfTotalMers());

  //  Guess how many mers we can fit into 512MB, then report how many chunks we need to do.

  uint32  merSize      = MSR->merSize();
  uint64  memoryLimit  = 700 * 1024 * 1024;
  uint64  perMer       = sizeof(kMerLite) + sizeof(dnode_t);
  uint64  mersPerBatch = memoryLimit / perMer;
  uint32  numBatches   = MSR->numberOfDistinctMers() / mersPerBatch;
  uint32  batch        = 0;

  dnode_t   *nodes     = new dnode_t  [mersPerBatch];
  kMerLite  *mers      = new kMerLite [mersPerBatch];

  if (MSR->numberOfDistinctMers() % mersPerBatch)
    numBatches++;

  fprintf(stderr, "perMer:  "uint64FMT" bytes ("uint64FMT" for kMerLite, "uint64FMT" for dnode_t.\n",
          perMer, (uint64)sizeof(kMerLite), (uint64)sizeof(dnode_t));
  fprintf(stderr, "We can fit "uint64FMT" mers into "uint64FMT"MB.\n", mersPerBatch, memoryLimit >> 20);
  fprintf(stderr, "So we need "uint32FMT" batches to verify the count.\n", numBatches);

  while (MSR->validMer()) {
    uint64          mersRemain = mersPerBatch;
    dict_t         *merDict    = dict_create(mersPerBatch, kMerLiteSort);

    batch++;

    //  STEP 1:  Insert mersPerBatch into the merDict
    //
    fprintf(stderr, "STEP 1 BATCH "uint32FMTW(2)":  Insert into merDict\n", batch);
    while (MSR->nextMer() && mersRemain) {
      mersRemain--;

      mers[mersRemain] = MSR->theFMer();

      //  initialize the node with the value, then insert the node
      //  into the tree using the key

      int32 val = (int32)MSR->theCount();
      dnode_init(&nodes[mersRemain], (void *)val);
      dict_insert(merDict, &nodes[mersRemain], &mers[mersRemain]);
    }

    //  STEP 2:  Stream the original file, decrementing the count
    //
    fprintf(stderr, "STEP 2 BATCH "uint32FMTW(2)":  Stream fasta\n", batch);
    seqStream    *CS = new seqStream(fastaName, true);
    merStream    *MS = new merStream(new kMerBuilder(merSize), CS);

    kMerLite       mer;
    dnode_t       *nod;

    while (MS->nextMer()) {
      mer = MS->theFMer();

      nod = dict_lookup(merDict, &mer);

      if (nod != 0L) {
        int32 val = (int32)dnode_get(nod);
        val--;
        dnode_put(nod, (void *)val);
      } else {
        //  Unless the whole meryl file fit into our merDict, we cannot warn if
        //  we don't find mers.
        //
        if (numBatches == 1) {
          char str[1024];
          fprintf(stderr, "Didn't find node for mer '%s'\n", mer.merToString(merSize, str));
        }
      }
    }

    delete MS;
    delete CS;

    //  STEP 3:  Check every node in the tree to make sure that the counts
    //  are exactly zero.
    //
    fprintf(stderr, "STEP 3 BATCH "uint32FMTW(2)":  Check\n", batch);
    nod = dict_first(merDict);
    while (nod) {
      int32           val = (int32)dnode_get(nod); 
      kMerLite const  *nodmer = (kMerLite const *)dnode_getkey(nod);

      if (val != 0) {
        char str[1024];
        fprintf(stderr, "Got count "int32FMT" for mer '%s'\n",
                val,
                nodmer->merToString(merSize, str));
      }

      nod = dict_next(merDict, nod);
    }


    //  STEP 4:  Destroy the dictionary.
    //
    fprintf(stderr, "STEP 4 BATCH "uint32FMTW(2)":  Destroy\n", batch);
    while ((nod = dict_first(merDict)))
      dict_delete(merDict, nod);
    dict_destroy(merDict);
  }
}
int
main(int argc, char **argv) {
  uint32       pNum   = 0;
  uint32       pAlloc = 8388608;
  uint32       estID  = ~uint32ZERO;

  bool        *found  = 0L;

  //  From fixPolishesIID.c
  IIDdict = 0L;
  SEQdict = 0L;
  GENdict = 0L;

  //  Incorporated from sortPolishes
  mergeFilesLen   = 0;
  mergeFilesMax   = sysconf(_SC_OPEN_MAX);
  mergeFiles      = new FILE *       [mergeFilesMax];
  mergeNames      = new char *       [mergeFilesMax];
  mergePolishes   = new sim4polish * [mergeFilesMax];

  //  Default to printing stats on stdout.
  sFile = stdout;

  int arg = 1;
  while (arg < argc) {
    if        (strcmp(argv[arg], "-n") == 0) {
      pAlloc = strtouint32(argv[++arg], 0L);

    } else if (strcmp(argv[arg], "-fpart") == 0) {
      arg++;
      fprintf(stderr, "reading query deflines from '%s'\n", argv[arg]);
      IIDdict = dict_create(DICTCOUNT_T_MAX, headerCompare);
      addToDict(IIDdict, argv[arg]);
    } else if (strcmp(argv[arg], "-g") == 0) {
      ++arg;
      fprintf(stderr, "reading genomic deflines from '%s'\n", argv[arg]);
      GENdict = dict_create(DICTCOUNT_T_MAX, headerCompare);
      addToDict(GENdict, argv[arg]);
    } else if (strcmp(argv[arg], "-F") == 0) {
      ++arg;
      fprintf(stderr, "reading query deflines from '%s'\n", argv[arg]);
      SEQdict = dict_create(DICTCOUNT_T_MAX, headerCompare);
      addToDict(SEQdict, argv[arg]);
    } else if (strcmp(argv[arg], "-f") == 0) {
      ++arg;
      SEQ = new seqCache(argv[arg]);
    } else if (strcmp(argv[arg], "-q") == 0) {
      ++arg;
      QLT = new seqCache(argv[arg]);

    } else if (strcmp(argv[arg], "-filter") == 0) {
      filter = atof(argv[++arg]);
      doFiltering = true;
    } else if (strcmp(argv[arg], "-output") == 0) {
      char  cmd[1024] = {0};
      errno = 0;
      ++arg;
      if (strcmp(argv[arg] + strlen(argv[arg]) - 4, ".bz2") == 0) {
        sprintf(cmd, "bzip2 -1c > %s", argv[arg]);
        oFile = popen(cmd, "w");
        oFileIsPipe = 1;
      } else if (strcmp(argv[arg] + strlen(argv[arg]) - 3, ".gz") == 0) {
        sprintf(cmd, "gzip -1c > %s", argv[arg]);
        oFile = popen(cmd, "w");
        oFileIsPipe = 1;
      } else {
        fprintf(stderr, "Got %s, not .bz2 not .gz!\n", argv[arg]);
        exit(1);
      }
      if (errno)
        fprintf(stderr, "Failed to open '%s': %s\n", cmd, strerror(errno));
      doFiltering = true;
    } else if (strcmp(argv[arg], "-scores") == 0) {
      errno = 0;
      sFile = fopen(argv[++arg], "w");
      if (errno)
        fprintf(stderr, "Failed to open '%s': %s\n", argv[arg-1], strerror(errno));
      doFiltering = true;
    } else if (strcmp(argv[arg], "-unique") == 0) {
      char  cmd[1024] = {0};
      errno = 0;
      arg++;
      if (strcmp(argv[arg] + strlen(argv[arg]) - 4, ".bz2") == 0)
        sprintf(cmd, "bzip2 -1c > %s", argv[arg]);
      else if (strcmp(argv[arg] + strlen(argv[arg]) - 3, ".gz") == 0)
        sprintf(cmd, "gzip -1c > %s", argv[arg]);
      else
        sprintf(cmd, "cat > %s", argv[arg]);
      uFile = popen(cmd, "w");
      if (errno)
        fprintf(stderr, "Failed to open '%s': %s\n", cmd, strerror(errno));
      doFiltering = true;

    } else if (strncmp(argv[arg], "-M", 2) == 0) {
      arg++;
      while ((arg < argc) && (fileExists(argv[arg]))) {
        if (mergeFilesLen >= mergeFilesMax) {
          fprintf(stderr, "%s: ERROR!  Too many input files!  Should be less than %d\n", argv[0], mergeFilesMax);
          exit(1);
        }
        mergeNames[mergeFilesLen]   = argv[arg];
        mergeFiles[mergeFilesLen++] = openFile(argv[arg], "r");
        arg++;
      }
      arg--;

    } else {
      fprintf(stderr, "unknown option: %s\n", argv[arg]);
    }
    arg++;
  }


  if (doFiltering) {
    if (uFile == 0L)
      fprintf(stderr, "ERROR:  -unique is required\n"), exit(1);
    if (sFile == 0L)
      fprintf(stderr, "ERROR:  -scores is required\n"), exit(1);
    if ((filter < 0.0) || (filter > 1.0))
      fprintf(stderr, "ERROR:  -filter value of %f invalid.  0 <= F <= 100.\n", filter), exit(1);
  }


  if ((IIDdict == 0L) || (SEQdict == 0L) || (GENdict == 0L)) {
    fprintf(stderr, "WARNING!  No sequence dictionaries, NOT FIXING IIDs!  (supply -fpart, -f and -g)\n");
  }


  if ((SEQ == 0L) || (QLT == 0L)) {
    fprintf(stderr, "I need -f and -q\n");
    exit(1);
  }

  //  We no longer require that input polishes be sorted increasingly;
  //  now they only must be grouped.  This remembers if we've seen a
  //  match or not.  At the end, we'll analyze() those we haven't done
  //  already.
  //
  found = new bool [ SEQ->getNumberOfSequences() ];
  for (uint32 i=0; i<SEQ->getNumberOfSequences(); i++)
    found[i] = false;


  //  Initialize the merge -- if no merge files, nothing done!
  //
  for (int i=0; i<mergeFilesLen; i++) {
    mergePolishes[i] = new sim4polish(mergeFiles[i]);
    fixIID(mergePolishes[i], IIDdict);
  }


  //  Read polishes, picking the best when we see a change in the
  //  estID.

  sim4polish **p = new sim4polish * [pAlloc];
  sim4polish  *q;

  while ((q = nextPolish()) != 0L) {

    if ((q->_estID != estID) && (pNum > 0)) {
      //fprintf(stderr, "PickBest for estID "uint32FMT"\n", estID);

      found[estID] = true;
      pickBest(p, pNum);
      pNum  = 0;
    }

    if (pNum >= pAlloc) {
      sim4polish **P = new sim4polish * [pAlloc * 2];
      memcpy(p, P, sizeof(sim4polish *) * pAlloc);
      delete [] p;
      p = P;
      pAlloc *= 2;
    }

    p[pNum++] = q;
    estID     = q->_estID;
  }

  if (pNum > 0) {
    found[estID] = true;
    pickBest(p, pNum);
  }

  //  Attempt cleanup
  //
  for (int i=0; i<mergeFilesLen; i++)
    closeFile(mergeFiles[i], mergeNames[i]);

  for (estID=0; estID < SEQ->getNumberOfSequences(); estID++)
    if (found[estID] == false)
      analyze(estID, 0, SEQ->getSequenceLength(estID), SEQ->getSequenceLength(estID), true, 'M');

  delete [] mergeFiles;
  delete [] mergeNames;
  delete [] mergePolishes;

  if (oFile)  pclose(oFile);
  if (uFile)  pclose(uFile);
  if (sFile)  fclose(sFile);

  fprintf(stderr, "Uni:"uint32FMTW(8)" Con:"uint32FMTW(8)" (T:"uint32FMTW(8)" M:"uint32FMTW(8)" I:"uint32FMTW(8)" S:"uint32FMTW(8)" N:"uint32FMTW(8)") Inc:"uint32FMTW(8)" -- Save:"uint32FMTW(8)" Lost:"uint32FMTW(8)"\n",
          statOneMatch,
          statConsistent, consistentTie, consistentMatches, consistentIdentity, consistentTooShort, consistentNot,
          statInconsistent,
          statUnique, statLost);
  fprintf(stderr, "total:  LQ:"uint32FMT" MQ:"uint32FMT" RQ:"uint32FMT"\n",
          totLQ, totMQ, totRQ);

  return(0);
}
Beispiel #24
0
/// Convert all raw PAs in the group into a single set of "cooked" PAs
/// under the leader.
/// @param[in] ca       the object pointer
void
ca_coalesce(ca_o ca)
{
    dict_t *dict_raw, *dict_cooked;
    dnode_t *dnpr, *dnpc, *next;

    dict_raw = ca->ca_raw_pa_dict;

    assert(!ca->ca_cooked_pa_dict);
    if ((dict_cooked = dict_create(DICTCOUNT_T_MAX, pa_cmp_by_pathname))) {
        ca->ca_cooked_pa_dict = dict_cooked;
    } else {
        putil_syserr(2, "dict_create()");
    }

    for (dnpr = dict_first(dict_raw); dnpr;) {
        pa_o raw_pa, ckd_pa;

        next = dict_next(dict_raw, dnpr);

        raw_pa = (pa_o)dnode_getkey(dnpr);

        _ca_verbosity_pa(raw_pa, ca, "COALESCING");

        // All data is in the key - that's why the value can be null.
        if ((dnpc = dict_lookup(dict_cooked, raw_pa))) {
            int keep_cooked = 0;

            ckd_pa = (pa_o)dnode_getkey(dnpc);

            if (!pa_is_read(raw_pa) && !pa_is_read(ckd_pa)) {
                moment_s raw_timestamp, ckd_timestamp;

                // If they're both destructive ops (non-read) then we
                // need to consider timestamps and use the later one.

                if (pa_has_timestamp(raw_pa) && pa_has_timestamp(ckd_pa)) {
                    // If the PA's have their own timestamps, use them.
                    raw_timestamp = pa_get_timestamp(raw_pa);
                    ckd_timestamp = pa_get_timestamp(ckd_pa);
                } else {
                    // Otherwise key off the file times. This is for
                    // support of "dummy" PAs as used in shopping.
                    raw_timestamp = pa_get_moment(raw_pa);
                    ckd_timestamp = pa_get_moment(ckd_pa);
                }

                if (moment_cmp(raw_timestamp, ckd_timestamp, NULL) <= 0) {
                    // Cooked write op is newer and can stay.
                    keep_cooked = 1;
                }
            } else if (pa_is_read(raw_pa)) {
                // There's no point replacing a read with another read,
                // so regardless of whether the current cooked PA is a
                // read or write, it can stay.
                keep_cooked = 1;
            } else {
                // A write always beats a read.
            }

            if (!keep_cooked) {
                dict_delete(dict_cooked, dnpc);
                dnode_destroy(dnpc);
                _ca_verbosity_pa(ckd_pa, ca, "REMOVING");
                pa_destroy(ckd_pa);
                if (!(dnpc = dnode_create(NULL))) {
                    putil_syserr(2, "dnode_create()");
                }
                ckd_pa = pa_copy(raw_pa);
                dict_insert(dict_cooked, dnpc, ckd_pa);
            }
        } else {
            if (!(dnpc = dnode_create(NULL))) {
                putil_syserr(2, "dnode_create()");
            }
            ckd_pa = pa_copy(raw_pa);
            dict_insert(dict_cooked, dnpc, ckd_pa);
        }

        // Clean up the raw set as we move PAs to the cooked one.
        dict_delete(dict_raw, dnpr);
        dnode_destroy(dnpr);
        dnpr = next;
    }

    return;
}
Beispiel #25
0
/* Parses and executes the AGGREGATE procedure. */
int
cmd_aggregate (struct lexer *lexer, struct dataset *ds)
{
  struct dictionary *dict = dataset_dict (ds);
  struct agr_proc agr;
  struct file_handle *out_file = NULL;
  struct casereader *input = NULL, *group;
  struct casegrouper *grouper;
  struct casewriter *output = NULL;

  bool copy_documents = false;
  bool presorted = false;
  bool saw_direction;
  bool ok;

  memset(&agr, 0 , sizeof (agr));
  agr.missing = ITEMWISE;
  agr.src_dict = dict;
  subcase_init_empty (&agr.sort);

  /* OUTFILE subcommand must be first. */
  lex_match (lexer, T_SLASH);
  if (!lex_force_match_id (lexer, "OUTFILE"))
    goto error;
  lex_match (lexer, T_EQUALS);
  if (!lex_match (lexer, T_ASTERISK))
    {
      out_file = fh_parse (lexer, FH_REF_FILE, dataset_session (ds));
      if (out_file == NULL)
        goto error;
    }

  if (out_file == NULL && lex_match_id (lexer, "MODE"))
    {
      lex_match (lexer, T_EQUALS);
      if (lex_match_id (lexer, "ADDVARIABLES"))
	{
	  agr.add_variables = true;

	  /* presorted is assumed in ADDVARIABLES mode */
	  presorted = true;
	}
      else if (lex_match_id (lexer, "REPLACE"))
	{
	  agr.add_variables = false;
	}
      else
	goto error;
    }

  if ( agr.add_variables )
    agr.dict = dict_clone (dict);
  else
    agr.dict = dict_create (dict_get_encoding (dict));

  dict_set_label (agr.dict, dict_get_label (dict));
  dict_set_documents (agr.dict, dict_get_documents (dict));

  /* Read most of the subcommands. */
  for (;;)
    {
      lex_match (lexer, T_SLASH);

      if (lex_match_id (lexer, "MISSING"))
	{
	  lex_match (lexer, T_EQUALS);
	  if (!lex_match_id (lexer, "COLUMNWISE"))
	    {
	      lex_error_expecting (lexer, "COLUMNWISE", NULL);
              goto error;
	    }
	  agr.missing = COLUMNWISE;
	}
      else if (lex_match_id (lexer, "DOCUMENT"))
        copy_documents = true;
      else if (lex_match_id (lexer, "PRESORTED"))
        presorted = true;
      else if (lex_force_match_id (lexer, "BREAK"))
	{
          int i;

	  lex_match (lexer, T_EQUALS);
          if (!parse_sort_criteria (lexer, dict, &agr.sort, &agr.break_vars,
                                    &saw_direction))
            goto error;
          agr.break_var_cnt = subcase_get_n_fields (&agr.sort);

	  if  (! agr.add_variables)
	    for (i = 0; i < agr.break_var_cnt; i++)
	      dict_clone_var_assert (agr.dict, agr.break_vars[i]);

          /* BREAK must follow the options. */
          break;
	}
      else
        goto error;

    }
  if (presorted && saw_direction)
    msg (SW, _("When PRESORTED is specified, specifying sorting directions "
               "with (A) or (D) has no effect.  Output data will be sorted "
               "the same way as the input data."));

  /* Read in the aggregate functions. */
  lex_match (lexer, T_SLASH);
  if (!parse_aggregate_functions (lexer, dict, &agr))
    goto error;

  /* Delete documents. */
  if (!copy_documents)
    dict_clear_documents (agr.dict);

  /* Cancel SPLIT FILE. */
  dict_set_split_vars (agr.dict, NULL, 0);

  /* Initialize. */
  agr.case_cnt = 0;

  if (out_file == NULL)
    {
      /* The active dataset will be replaced by the aggregated data,
         so TEMPORARY is moot. */
      proc_cancel_temporary_transformations (ds);
      proc_discard_output (ds);
      output = autopaging_writer_create (dict_get_proto (agr.dict));
    }
  else
    {
      output = any_writer_open (out_file, agr.dict);
      if (output == NULL)
        goto error;
    }

  input = proc_open (ds);
  if (!subcase_is_empty (&agr.sort) && !presorted)
    {
      input = sort_execute (input, &agr.sort);
      subcase_clear (&agr.sort);
    }

  for (grouper = casegrouper_create_vars (input, agr.break_vars,
                                          agr.break_var_cnt);
       casegrouper_get_next_group (grouper, &group);
       casereader_destroy (group))
    {
      struct casereader *placeholder = NULL;
      struct ccase *c = casereader_peek (group, 0);

      if (c == NULL)
        {
          casereader_destroy (group);
          continue;
        }

      initialize_aggregate_info (&agr);

      if ( agr.add_variables )
	placeholder = casereader_clone (group);

      {
	struct ccase *cg;
	for (; (cg = casereader_read (group)) != NULL; case_unref (cg))
	  accumulate_aggregate_info (&agr, cg);
      }


      if  (agr.add_variables)
	{
	  struct ccase *cg;
	  for (; (cg = casereader_read (placeholder)) != NULL; case_unref (cg))
	    dump_aggregate_info (&agr, output, cg);

	  casereader_destroy (placeholder);
	}
      else
	{
	  dump_aggregate_info (&agr, output, c);
	}
      case_unref (c);
    }
  if (!casegrouper_destroy (grouper))
    goto error;

  if (!proc_commit (ds))
    {
      input = NULL;
      goto error;
    }
  input = NULL;

  if (out_file == NULL)
    {
      struct casereader *next_input = casewriter_make_reader (output);
      if (next_input == NULL)
        goto error;

      dataset_set_dict (ds, agr.dict);
      dataset_set_source (ds, next_input);
      agr.dict = NULL;
    }
  else
    {
      ok = casewriter_destroy (output);
      output = NULL;
      if (!ok)
        goto error;
    }

  agr_destroy (&agr);
  fh_unref (out_file);
  return CMD_SUCCESS;

error:
  if (input != NULL)
    proc_commit (ds);
  casewriter_destroy (output);
  agr_destroy (&agr);
  fh_unref (out_file);
  return CMD_CASCADING_FAILURE;
}
Beispiel #26
0
static void mms_queue_run(char *dir, 
			  int (*deliver)(MmsEnvelope *), 
			  double sleepsecs, int num_threads, int *rstop)
{
	int i, qstop = 0;
	List *stack = gwlist_create();
//	static struct Qthread_t *tlist;
	struct Qthread_t *tlist;
	
	debug("", 0, "mms_queue_run: %s", dir);
	
	max_live_exceeded_warning_issued = 0;
	
	number_of_threads = num_threads;
    gw_assert(num_threads>0);

	if (tlists == NULL)
		tlists = dict_create(10, NULL);
		
	Octstr *odir = octstr_create(dir);
	
	tlist = gw_malloc(num_threads*sizeof tlist[0]);
	dict_put(tlists, odir, tlist);
	gw_assert(tlist == dict_get(tlists,odir));
	
	debug("",0,"tlist allocated at addr %p", &tlist[0]);
	octstr_destroy(odir);
     
	for (i = 0; i<num_threads; i++) { /* Create threads for sending. */
		debug("",0,"%s tlist[%d] has addr: %p", dir, i, &tlist[i]);
		tlist[i].l = gwlist_create();
		gwlist_add_producer(tlist[i].l);
		tlist[i].deliver = deliver;
		gwthread_create((gwthread_func_t *)tdeliver, &tlist[i]);
	}

	i = 0;  /* For stepping through above array. */
	do { 
		Octstr *xdir = NULL;	  
		gwlist_append(stack, octstr_create("")); /* Put initial dir on there. */

		while (!*rstop && (xdir = gwlist_extract_first(stack)) != NULL) {
			int ret = run_dir(dir, octstr_get_cstr(xdir), tlist, num_threads, &i, stack);
			octstr_destroy(xdir);
			xdir = NULL;
			if (ret < 0) {		    
				if (ret <= -2)
					qstop = 1;
				goto qloop;
			}
		}

		octstr_destroy(xdir);
		if (*rstop) 
			break;
		qloop:
		gwthread_sleep(sleepsecs);
	} while (!qstop);

	/* We are out of the queue, time to go away. */
	for (i = 0; i<num_threads; i++)
		if (tlist[i].l)
			gwlist_remove_producer(tlist[i].l);
	gwthread_join_every((gwthread_func_t *)tdeliver); /* Wait for them all to terminate. */
  
	for (i = 0; i<num_threads; i++)
		if (tlist[i].l)
			gwlist_destroy(tlist[i].l,NULL); /* Final destroy if needed. */
			
	gw_free(tlist);
	gwlist_destroy(stack, (gwlist_item_destructor_t *)octstr_destroy);
	return;
}
Beispiel #27
0
int main(int argc, char **args){
    if(argc != 3){
        fprintf(stderr,"%s RECV SEND\n",args[0]);
        exit(1);
    }
    signal(SIGINT,&call_for_stop);
    
    bstring pull_addr = bfromcstr(args[1]);
    bstring pub_addr  = bfromcstr(args[2]);

    mongrel2_ctx *ctx = mongrel2_init(1); // Yes for threads?

    mongrel2_socket *pull_socket = mongrel2_pull_socket(ctx);
    mongrel2_connect(pull_socket, bdata(pull_addr));

    pub_socket = mongrel2_pub_socket(ctx);
    mongrel2_connect(pub_socket, bdata(pub_addr));
mongrel2_set_identity(pub_socket, bdata(&SENDER) );

    mongrel2_request *request;

    // Polling is done to show how to do a clean shutdown
    int poll_response;
    zmq_pollitem_t socket_tracker;
    socket_tracker.socket = pull_socket->zmq_socket;
    socket_tracker.events = ZMQ_POLLIN;

    // Let's try out some ADT goodness
    dict_t* dict = dict_create(DICTCOUNT_T_MAX, compare_session);
    dict_set_allocator(dict, alloc_dict, free_dict, NULL);

    dnode_t* tempnode = NULL;
    m2_ws_session_data *counter = NULL;
    int retval = 0;
    while(shutdown != 1){
        poll_response = zmq_poll(&socket_tracker,1,500*1000);
        if(poll_response > 0){
            request = mongrel2_recv(pull_socket);
            fprintf(stdout,"got something...\n");

            if(request != NULL && mongrel2_request_for_disconnect(request) != 1){
                m2_ws_session_id* incoming = calloc(1,sizeof(m2_ws_session_id));
                incoming->req = request;
                printf("Looking at incoming->conn_id = %d\n",incoming->req->conn_id);
                tempnode = dict_lookup(dict,incoming);

                if(tempnode == NULL){
                    mongrel2_ws_reply_upgrade(request,pub_socket);
                    counter = calloc(1,sizeof(m2_ws_session_data));
                    counter->times_seen = 0;
                    retval = dict_alloc_insert(dict,incoming,counter);
                    assert(retval == 1);
                } else {
                    free(incoming);
                    counter = dnode_get(tempnode);
                    counter->times_seen += 1;
                }

                if(blength(request->body) > 0){
                    if(tempnode && mongrel2_ws_frame_get_fin(blength(request->body),
                                                             (uint8_t*)bdata(request->body))){
                        printf("Hey, it's a close\n");
                        dict_delete_free(dict,tempnode);
                        mongrel2_disconnect(pub_socket,request);
                    }
                } else {
                    bstring randmsg = genrandmsg();
                    mongrel2_ws_reply(pub_socket,request,randmsg);
                }
                

                printf("FYI: we've got %ld entries\n",dict_count(dict));
            } else {
                fprintf(stdout,"Connection %d disconnected\n", request->conn_id);
            }
        } else if (poll_response < 0){
            fprintf(stdout, "Error on poll!");
            shutdown = 1;
        }
    }
    // bstring msg = bformat("{\"msg\" : \"hi there %d\"}", request->conn_id);
    // fprintf(stdout,"Sending new msg: '%*s'",blength(msg),bdata(msg));
    // mongrel2_ws_reply(pub_socket,request,msg);
    // bdestroy(msg);
    // mongrel2_request_finalize(request);
    // mongrel2_reply(pub_socket,request,bfromcstr(""));
    
    bdestroy(pull_addr);
    bdestroy(pub_addr);

    mongrel2_close(pull_socket);
    mongrel2_close(pub_socket);
    mongrel2_deinit(ctx);
    fprintf(stdout,"\nClean shutdown done! Thanks for playing!\n");
    return 0;
}
Beispiel #28
0
int main() {
	size_t i = 0, m, max=100000;
	char buf[128];
	struct timeval beg, end;
	long sec ;
	dict *d = dict_create(&opts);
	for(m = 0; m < 3; m++) {
		for(i = 0; i < max; i++) {
			memset(buf, 0, sizeof(buf));
			snprintf(buf, sizeof(buf), ";;%lu;;", i);
			dict_add(d, buf, buf);
		}
		printf("add %d, %d %llu\n", DICT_USED(d), DICT_CAP(d),used_mem());
		for(i = 0; i < max; i++) {
			dict_entry *entry;
			memset(buf, 0, sizeof(buf));
			snprintf(buf, sizeof(buf), ";;%lu;;", i);
			entry = dict_find(d, buf);
			if(strcmp((char*)entry->value, (char*)buf) != 0)
				abort();
		}
		printf("find %d, %d %llu\n", DICT_USED(d), DICT_CAP(d),used_mem());
		for(i = 0; i < max; i++) {
			memset(buf, 0, sizeof(buf));
			snprintf(buf, sizeof(buf), ";;%lu;;", i);
			dict_replace(d, buf, buf);
		}
		printf("prelace %d, %d %llu\n", DICT_USED(d), DICT_CAP(d),used_mem());
		for(i = 0; i < max; i++) {
			memset(buf, 0, sizeof(buf));
			snprintf(buf, sizeof(buf), ";;%lu;;", i);
			dict_del(d, buf);
		}
		printf("delete %d, %d %llu\n", DICT_USED(d), DICT_CAP(d),used_mem());
	}
	dict_expand(d, max*2);
	printf("%d, %d %llu\n", DICT_USED(d), DICT_CAP(d),used_mem());
	printf("%d %llu\n", DICT_CAP(d),used_mem());
	dict_destroy(d);
	printf("%llu\n", used_mem());


	d = dict_create(&opts);

	gettimeofday(&beg, NULL);
	for(i = 0; i < 1000000; i++) {
		memset(buf, 0, sizeof(buf));
		snprintf(buf, sizeof(buf), "%lu", i);
		dict_add(d, buf, buf);
	}
	gettimeofday(&end, NULL);
	sec = (end.tv_sec - beg.tv_sec)*1000;
	sec += (end.tv_usec - beg.tv_usec)/1000;
	printf("add msec %ld, mm: %llu\n", sec, used_mem());

	gettimeofday(&beg, NULL);
	dict_iterator *iter = dict_get_iterator(d);
	dict_entry *entry = NULL;
	i = 0;
	while((entry = dict_iterator_next(iter)) != NULL){
		i++;
	}
	dict_iterator_destroy(iter);
	gettimeofday(&end, NULL);
	sec = (end.tv_sec - beg.tv_sec)*1000;
	sec += (end.tv_usec - beg.tv_usec)/1000;
	printf("iterator msec %ld, times: %lu mm: %llu\n", sec, i, used_mem());

	gettimeofday(&beg, NULL);
	for(i = 0; i < 1000000; i++) {
		memset(buf, 0, sizeof(buf));
		snprintf(buf, sizeof(buf), "%lu", i);
		dict_del(d, buf);
	}
	gettimeofday(&end, NULL);
	sec = (end.tv_sec - beg.tv_sec)*1000;
	sec += (end.tv_usec - beg.tv_usec)/1000;
	printf("remove msec %ld, mm: %llu\n", sec, used_mem());

	dict_destroy(d);
	printf("%llu\r\n", used_mem());


	return 0;
}