END_TEST
START_TEST(test_dict_add3)
{
TestData t, *pt;
t.ivalue = 1;
Dict *pdict = dict_create();
dict_add(pdict, SIZEOF("key1"), (char*)&t, sizeof(TestData));
pt = (TestData*)dict_get(pdict, SIZEOF("key1"));
ck_assert_int_eq(pt->ivalue, 1);
t.ivalue = 2;
dict_add(pdict, SIZEOF("key1"), (char*)&t, sizeof(TestData));
pt = (TestData*)dict_get(pdict, SIZEOF("key1"));
ck_assert_int_eq(pt->ivalue, 1);
}
void _sql_init(void) {
if (DBConnection < 0) {
logging_register_module(sql);
if (!_drivers) {
_driver_mutex = mutex_create();
_drivers = dict_create(NULL);
dict_set_key_type(_drivers, &type_str);
dict_set_data_type(_drivers, &type_int);
}
if (ErrorSQL < 1) {
exception_register(ErrorSQL);
}
typedescr_register_with_methods(DBConnection, dbconn_t);
typedescr_register(DBTransaction, tx_t);
}
assert(DBConnection);
}
void smpp_route_rebuild_database(SMPPServer *smpp_server) {
info(0, "Rebuilding database routes");
SMPPRouting *smpp_routing = smpp_server->routing;
List *inbound_routes = smpp_database_get_routes(smpp_server, SMPP_ROUTE_DIRECTION_INBOUND, NULL); /* Only inbound are built, outbound are built when ESME's connect */
List *old_inbound;
Dict *old_outbound;
gw_rwlock_wrlock(smpp_routing->lock);
old_inbound = smpp_routing->inbound_routes;
old_outbound = smpp_routing->outbound_routes;
smpp_routing->inbound_routes = inbound_routes;
smpp_routing->outbound_routes = dict_create(1024, (void(*)(void *))smpp_outbound_routes_destroy); /* Just reset, they will repopulate on their own */
gw_rwlock_unlock(smpp_routing->lock);
gwlist_destroy(old_inbound, (void(*)(void *))smpp_route_destroy);
dict_destroy(old_outbound);
}
int cxml_node_addnode(CLOG_INFO* info, CXMLNODE *parent, CXMLNODE *child)
{
int return_value=0;
assert(NULL!=parent);
assert(NULL!=child);
// if the sub node is NULL create a new sub table
if(NULL==parent->sub) {
clog( info, CTRACE, "XML: xml_node_addnode(), creating sub node");
parent->sub = dict_create(DICTCOUNT_T_MAX, (dict_comp_t)strcmp);
// allow duplicates
dict_allow_dupes(parent->sub);
}
// if the previous stuff worked, add the name/value pair
if(NULL!=parent->sub && NULL!=child->name && NULL!=child->name->string) {
dnode_t *node = (parent->sub)->dict_allocnode((parent->sub)->dict_context);
char *key = strdup(child->name->string);
clog( info, CTRACE, "XML: xml_node_adddata(), Saving sub node name=\"%.80s\"", child->name->string);
if (node) {
dnode_init(node, child);
dict_insert(parent->sub, node, key);
// let the child know who they are
child->me = node;
// success!
return_value=1;
}
}
// let the child know whow their parent is
if(NULL!=parent && NULL!=child) {
child->parent = parent;
}
return return_value;
}
static int tolua_dict_set_usertype(lua_State * L, int type)
{
dict self = (dict)tolua_tousertype(L, 1, 0);
const char *name = tolua_tostring(L, 2, 0);
unit *value = tolua_tousertype(L, 3, 0);
attrib *a = a_find(*self, &at_dict);
variant val;
val.v = value;
for (; a && a->type == &at_dict; a = a->next) {
if (strcmp(dict_name(a), name) == 0) {
dict_set(a, type, val);
return 0;
}
}
a = a_add(self, dict_create(name, type, val));
return 0;
}
int init_update(void)
{
dict_types type;
memset(&type, 0, sizeof(type));
type.hash_function = update_dict_hash_function;
type.key_compare = update_dict_key_compare;
type.key_dup = update_dict_key_dup;
type.key_destructor = update_dict_key_free;
type.val_dup = update_dict_val_dup;
type.val_destructor = update_dict_val_free;
dict_update = dict_create(&type, 64);
if (dict_update == NULL)
return -__LINE__;
nw_timer_set(&timer, 60, true, on_timer, NULL);
nw_timer_start(&timer);
return 0;
}
static int tolua_dict_set_number(lua_State * L)
{
dict self = (dict)tolua_tousertype(L, 1, 0);
const char *name = tolua_tostring(L, 2, 0);
lua_Number value = tolua_tonumber(L, 3, 0);
attrib *a = a_find(*self, &at_dict);
variant val;
val.f = (float)value;
for (; a && a->type == &at_dict; a = a->next) {
if (strcmp(dict_name(a), name) == 0) {
dict_set(a, TREAL, val);
return 0;
}
}
a = a_add(self, dict_create(name, TREAL, val));
return 0;
}
END_TEST
START_TEST(test_dict_add1)
{
TestData t;
t.ivalue = 1;
Dict *pdict = dict_create();
dict_add(pdict, SIZEOF("key1"), (char*)&t, sizeof(TestData));
ck_assert_int_eq(dict_get_count(pdict), 1);
TestData *t1 = (TestData*)dict_get(pdict, SIZEOF("key1"));
ck_assert_msg(t1 != &t, "dict did not make a copy of input data");
ck_assert_int_eq(t1->ivalue, 1);
dict_add(pdict, SIZEOF("key2"), (char*)&t, sizeof(TestData));
TestData *t2 = (TestData*)dict_get(pdict, SIZEOF("key2"));
ck_assert_msg(t1 != t2, "cannot add the same data with two different keys");
dict_free(pdict);
}
void testBasicDict() {
struct dict *d = dict_create();
dict_set(d, "name", "drmaa2");
dict_del(d, "name");
dict_set(d, "name", "drmaa2");
assert(dict_has(d, "name") != 0);
assert(dict_has(d, "language") == 0);
const char *v = dict_get(d, "name");
assert(strcmp(v, "drmaa2") == 0);
dict_set(d, "language", "c");
assert(dict_has(d, "language") != 0);
dict_del(d, "name");
assert(dict_has(d, "name") == 0);
v = dict_get(d, "language");
assert(strcmp(v, "c") == 0);
dict_free(d);
}
void cmd_init()
{
command_list = dict_create();
dict_set_free_funcs(command_list, NULL, (dict_free_f *)cmd_free_subcmds);
// Initialize our own commands
cmd_register_list(commands, NULL);
cmd_alias("exit", "quit", NULL);
// Initialize external commands
cmd_server_init();
cmd_link_init();
cmd_conf_init();
cmd_oper_init();
cmd_service_init();
cmd_forward_init();
cmd_feature_init();
cmd_jupe_init();
cmd_class_init();
cmd_pseudo_init();
cmd_client_init();
cmd_webirc_init();
}
size_t get_file(const char *file_path, struct dict **dicts)
{
char *token, buffer[1024];
size_t size = 0, alloc = 1;
struct dict *d = malloc(sizeof(struct dict));
FILE *f = fopen(file_path, "r");
while (fgets(buffer, sizeof(buffer), f) != NULL) {
if ((token = strchr(buffer, '#')) != NULL)
*token = '\0';
if ((token = strchr(buffer, '=')) == NULL)
continue;
if (size == alloc) {
alloc *= 2;
d = realloc(d, alloc * sizeof(struct dict));
}
*token++ = '\0';
while (*token == ' ' || *token == '\t')
token++;
dict_create(d + size++, buffer, strlen(buffer), token, strlen(token));
}
fclose(f);
return size;
}
TEST(mydict, dict)
{
dict_t* d = dict_create((void*)123);
int k, v;
dict_node_t* node = NULL;
int i = 0;
int sum = 0;
for (i = 0; i < 10000; i++) {
k = i, v = i;
node = dict_add(d, &k, sizeof(k), &v, sizeof(v));
ASSERT_TRUE(node != NULL);
sum += i;
}
for (i = 0; i < 10000; i++) {
node = dict_find(d, &k, sizeof(k));
ASSERT_EQ((long)dict_get_val(node), i);
}
i = 0;
for(node = dict_first(d); node; node = dict_next(d, node)) {
i += (long)dict_get_val(node);
}
ASSERT_EQ(i, sum);
for (i = 0; i < 10000; i++) {
k = i;
node = dict_delete(d, &k, sizeof(k));
ASSERT_EQ((long)dict_get_val(node), i);
}
ASSERT_TRUE(dict_empty(d));
dict_destroy(d);
}
int store_file_init(const Octstr *fname, long dump_freq)
{
/* Initialize function pointers */
store_messages = store_file_messages;
store_save = store_file_save;
store_save_ack = store_file_save_ack;
store_load = store_file_load;
store_dump = store_file_dump;
store_shutdown = store_file_shutdown;
store_status = store_file_status;
if (fname == NULL)
return 0; /* we are done */
if (octstr_len(fname) > (FILENAME_MAX-5))
panic(0, "Store file filename too long: `%s', failed to init.",
octstr_get_cstr(fname));
filename = octstr_duplicate(fname);
newfile = octstr_format("%s.new", octstr_get_cstr(filename));
bakfile = octstr_format("%s.bak", octstr_get_cstr(filename));
sms_dict = dict_create(1024, msg_destroy_item);
if (dump_freq > 0)
dump_frequency = dump_freq;
else
dump_frequency = BB_STORE_DEFAULT_DUMP_FREQ;
file_mutex = mutex_create();
active = 1;
loaded = gwlist_create();
gwlist_add_producer(loaded);
return 0;
}
/* Set the clip according to the currently
selected range in the data sheet */
static void
data_sheet_set_clip (PsppireSheet *sheet)
{
int i;
struct casewriter *writer ;
PsppireSheetRange range;
PsppireDataStore *ds;
struct case_map *map = NULL;
casenumber max_rows;
size_t max_columns;
gint row0, rowi;
gint col0, coli;
ds = PSPPIRE_DATA_STORE (psppire_sheet_get_model (sheet));
psppire_sheet_get_selected_range (sheet, &range);
col0 = MIN (range.col0, range.coli);
coli = MAX (range.col0, range.coli);
row0 = MIN (range.row0, range.rowi);
rowi = MAX (range.row0, range.rowi);
/* If nothing selected, then use active cell */
if ( row0 < 0 || col0 < 0 )
{
gint row, col;
psppire_sheet_get_active_cell (sheet, &row, &col);
row0 = rowi = row;
col0 = coli = col;
}
/* The sheet range can include cells that do not include data.
Exclude them from the range. */
max_rows = psppire_data_store_get_case_count (ds);
if (rowi >= max_rows)
{
if (max_rows == 0)
return;
rowi = max_rows - 1;
}
max_columns = dict_get_var_cnt (ds->dict->dict);
if (coli >= max_columns)
{
if (max_columns == 0)
return;
coli = max_columns - 1;
}
/* Destroy any existing clip */
if ( clip_datasheet )
{
casereader_destroy (clip_datasheet);
clip_datasheet = NULL;
}
if ( clip_dict )
{
dict_destroy (clip_dict);
clip_dict = NULL;
}
/* Construct clip dictionary. */
clip_dict = dict_create (dict_get_encoding (ds->dict->dict));
for (i = col0; i <= coli; i++)
dict_clone_var_assert (clip_dict, dict_get_var (ds->dict->dict, i));
/* Construct clip data. */
map = case_map_by_name (ds->dict->dict, clip_dict);
writer = autopaging_writer_create (dict_get_proto (clip_dict));
for (i = row0; i <= rowi ; ++i )
{
struct ccase *old = psppire_data_store_get_case (ds, i);
if (old != NULL)
casewriter_write (writer, case_map_execute (map, old));
else
casewriter_force_error (writer);
}
case_map_destroy (map);
clip_datasheet = casewriter_make_reader (writer);
data_sheet_update_clipboard (sheet);
}
int main(int argc,char *argv[],char *env[])
{
pdict = dict_create();
FILE *fp = fopen(FILE_NAME,"r");
if(fp == NULL)
{
perror("fopen");
return FAILED;
}
char english[1000];
char chinese[1000];
while(!feof(fp))
{
memset(english,0,sizeof(english));
memset(chinese,0,sizeof(chinese));
if(fgets(english,sizeof(english),fp) == NULL)
break;
*strchr(english,'\n') = 0;
if(fgets(chinese,sizeof(chinese),fp) == NULL)
break;
*strchr(chinese,'\n') = 0;
dict_insert(pdict,english,chinese);
}
signal(28,handle);
show_window();
move_xy(3,15);
char key;
while(1)
{
key = get_key();
switch(key)
{
case ESC:
clear_screen();
move_xy(1,1);
return 0;
case BACKSPACE:
if(buf_i > 0)
{
move_left(1);
putchar(' ');
fflush(stdout);
move_left(1);
buf_i--;
input_buf[buf_i] = 0;
show_word(pdict,input_buf);
}
break;
default:
if(buf_i <= win_y - 20 && isprint(key))
{
putchar(key);
fflush(stdout);
input_buf[buf_i] = key;
buf_i++;
input_buf[buf_i] = 0;
show_word(pdict,input_buf);
}
}
}
return 0;
}
static void run_smsbox(void *arg)
{
Boxc *newconn;
long sender;
Msg *msg;
List *keys;
Octstr *key;
gwlist_add_producer(flow_threads);
newconn = arg;
newconn->incoming = gwlist_create();
gwlist_add_producer(newconn->incoming);
newconn->retry = incoming_sms;
newconn->outgoing = outgoing_sms;
newconn->sent = dict_create(smsbox_max_pending, NULL);
newconn->pending = semaphore_create(smsbox_max_pending);
sender = gwthread_create(boxc_sender, newconn);
if (sender == -1) {
error(0, "Failed to start a new thread, disconnecting client <%s>",
octstr_get_cstr(newconn->client_ip));
goto cleanup;
}
/*
* We register newconn in the smsbox_list here but mark newconn as routable
* after identification or first message received from smsbox. So we can avoid
* a race condition for routable smsboxes (otherwise between startup and
* registration we will forward some messages to smsbox).
*/
gw_rwlock_wrlock(smsbox_list_rwlock);
gwlist_append(smsbox_list, newconn);
gw_rwlock_unlock(smsbox_list_rwlock);
gwlist_add_producer(newconn->outgoing);
boxc_receiver(newconn);
gwlist_remove_producer(newconn->outgoing);
/* remove us from smsbox routing list */
gw_rwlock_wrlock(smsbox_list_rwlock);
gwlist_delete_equal(smsbox_list, newconn);
if (newconn->boxc_id) {
dict_remove(smsbox_by_id, newconn->boxc_id);
}
gw_rwlock_unlock(smsbox_list_rwlock);
/*
* check if we in the shutdown phase and sms dequeueing thread
* has removed the producer already
*/
if (gwlist_producer_count(newconn->incoming) > 0)
gwlist_remove_producer(newconn->incoming);
/* check if we are still waiting for ack's and semaphore locked */
if (dict_key_count(newconn->sent) >= smsbox_max_pending)
semaphore_up(newconn->pending); /* allow sender to go down */
gwthread_join(sender);
/* put not acked msgs into incoming queue */
keys = dict_keys(newconn->sent);
while((key = gwlist_extract_first(keys)) != NULL) {
msg = dict_remove(newconn->sent, key);
gwlist_produce(incoming_sms, msg);
octstr_destroy(key);
}
gw_assert(gwlist_len(keys) == 0);
gwlist_destroy(keys, octstr_destroy_item);
/* clear our send queue */
while((msg = gwlist_extract_first(newconn->incoming)) != NULL) {
gwlist_produce(incoming_sms, msg);
}
cleanup:
gw_assert(gwlist_len(newconn->incoming) == 0);
gwlist_destroy(newconn->incoming, NULL);
gw_assert(dict_key_count(newconn->sent) == 0);
dict_destroy(newconn->sent);
semaphore_destroy(newconn->pending);
boxc_destroy(newconn);
/* wakeup the dequeueing thread */
gwthread_wakeup(sms_dequeue_thread);
gwlist_remove_producer(flow_threads);
}
int smpp_pdu_init(Cfg *cfg)
{
CfgGroup *grp;
List *l;
if (initialized)
return 0;
l = cfg_get_multi_group(cfg, octstr_imm("smpp-tlv"));
tlvs = gwlist_create();
tlvs_by_tag = dict_create(1024, (void(*)(void*))dict_destroy);
tlvs_by_name = dict_create(1024, (void(*)(void*))dict_destroy);
while (l != NULL && (grp = gwlist_extract_first(l)) != NULL) {
struct smpp_tlv *tlv;
Octstr *tmp, *smsc_id;
List *l2;
tlv = gw_malloc(sizeof(*tlv));
if ((tlv->name = cfg_get(grp, octstr_imm("name"))) == NULL) {
error(0, "SMPP: Unable to get name for smpp-tlv.");
smpp_tlv_destroy(tlv);
goto failed;
}
if (cfg_get_integer(&tlv->tag, grp, octstr_imm("tag")) == -1) {
error(0, "SMPP: Unable to get tag for smpp-tlv.");
smpp_tlv_destroy(tlv);
goto failed;
}
if (cfg_get_integer(&tlv->length, grp, octstr_imm("length")) == -1) {
error(0, "SMPP: Unable to get length for smpp-tlv.");
smpp_tlv_destroy(tlv);
goto failed;
}
if ((tmp = cfg_get(grp, octstr_imm("type"))) == NULL) {
error(0, "SMPP: Unable to get type for smpp-tlv.");
smpp_tlv_destroy(tlv);
goto failed;
}
if (octstr_str_case_compare(tmp, "octetstring") == 0)
tlv->type = SMPP_TLV_OCTETS;
else if (octstr_str_case_compare(tmp, "nulterminated") == 0)
tlv->type = SMPP_TLV_NULTERMINATED;
else if (octstr_str_case_compare(tmp, "integer") == 0)
tlv->type = SMPP_TLV_INTEGER;
else {
error(0, "SMPP: Unknown type for smpp-tlv: `%s'", octstr_get_cstr(tmp));
octstr_destroy(tmp);
smpp_tlv_destroy(tlv);
goto failed;
}
octstr_destroy(tmp);
/* put to all TLVs */
gwlist_produce(tlvs, tlv);
smsc_id = cfg_get(grp, octstr_imm("smsc-id"));
if (smsc_id != NULL) {
l2 = octstr_split(smsc_id, octstr_imm(";"));
octstr_destroy(smsc_id);
} else {
l2 = gwlist_create();
gwlist_produce(l2, octstr_create(DEFAULT_SMSC_ID));
}
while(l2 != NULL && (smsc_id = gwlist_extract_first(l2)) != NULL) {
Dict *tmp_dict;
debug("sms.smpp", 0, "adding smpp-tlv for smsc-id=%s", octstr_get_cstr(smsc_id));
tmp_dict = dict_get(tlvs_by_name, smsc_id);
if (tmp_dict == NULL) {
tmp_dict = dict_create(1024, NULL);
dict_put(tlvs_by_name, smsc_id, tmp_dict);
}
/* put into dict */
if (!dict_put_once(tmp_dict, tlv->name, tlv)) {
error(0, "SMPP: Double TLV name %s found.", octstr_get_cstr(tlv->name));
octstr_destroy(smsc_id);
goto failed;
}
tmp_dict = dict_get(tlvs_by_tag, smsc_id);
if (tmp_dict == NULL) {
tmp_dict = dict_create(1024, NULL);
dict_put(tlvs_by_tag, smsc_id, tmp_dict);
}
tmp = octstr_format("%ld", tlv->tag);
if (!dict_put_once(tmp_dict, tmp, tlv)) {
error(0, "SMPP: Double TLV tag %s found.", octstr_get_cstr(tmp));
gwlist_destroy(l2, octstr_destroy_item);
octstr_destroy(tmp);
octstr_destroy(smsc_id);
goto failed;
}
octstr_destroy(tmp);
octstr_destroy(smsc_id);
}
gwlist_destroy(l2, octstr_destroy_item);
}
gwlist_destroy(l, NULL);
initialized = 1;
return 0;
failed:
gwlist_destroy(tlvs, (void(*)(void*))smpp_tlv_destroy);
dict_destroy(tlvs_by_tag);
dict_destroy(tlvs_by_name);
return -1;
}
mCfg *mms_cfg_read(Octstr *file)
{
Octstr *sf;
List *lines;
int i, n;
mCfg *cfg;
mCfgGrp *grp = NULL;
int skip = 0;
gw_assert(file);
if ((sf = octstr_read_file(octstr_get_cstr(file))) == NULL) {
mms_error(errno, "mms_cfg", NULL, "failed to read config from `%s'", octstr_get_cstr(file));
return NULL;
}
cfg = gw_malloc(sizeof *cfg);
cfg->file = octstr_duplicate(file);
cfg->grps = dict_create(7, NULL);
cfg->xcfg = NULL;
cfg->cfg_funcs = NULL;
lines = octstr_split(sf, octstr_imm("\n"));
for (i = 0, n = gwlist_len(lines); i < n; i++) {
Octstr *current = gwlist_get(lines,i);
int pos;
octstr_strip_blanks(current);
if (octstr_len(current) == 0) { /* end of group. */
grp = NULL;
skip = 0;
continue;
} else if (skip || octstr_get_char(current, 0) == '#')
continue;
if ((pos = octstr_search_char(current, '=',0)) > 0) {
/* a field name. first see if start of grp */
Octstr *field = octstr_copy(current,0,pos);
Octstr *value = octstr_copy(current,pos+1,octstr_len(current));
octstr_strip_blanks(field);
fixup_value(value, i+1);
#if 0
mms_info(0, "mms_cfg", NULL, "field/value: [%s - %s]", octstr_get_cstr(field),
octstr_get_cstr(value));
#endif
if (octstr_str_case_compare(field, "group") == 0)
if (grp == NULL) { /* grp name. */
int ismulti = is_multigroup(value);
if (ismulti < 0) {
mms_info(0, "mms_cfg", NULL, "Skipping unknown group `%s' at line %d of conf file",
octstr_get_cstr(value), i+1);
skip = 1;
} else {
grp = gw_malloc(sizeof *grp);
grp->name = octstr_duplicate(value);
grp->fields = dict_create(23, (void (*)(void *))octstr_destroy);
if (ismulti) {
List *l = dict_get(cfg->grps, value);
if (l == NULL) {
l = gwlist_create();
dict_put(cfg->grps, value, l);
}
gwlist_append(l, grp);
} else if (dict_put_once(cfg->grps, value, grp) == 0)
panic(0, "Group `%s' [at line %d] cannot appear more "
"than once in config!",
octstr_get_cstr(value), i+1);
}
} else
panic(0,"`group' is an illegal field name "
"within a group at line %d in config file!",
i+1);
else if (grp) /* an ordinary field name. */
check_and_add_field(grp, field, value,i+1);
else
panic(0, "A group must begin with a `group = group_name' "
"clause [at line %d in config file]", i+1);
octstr_destroy(field);
octstr_destroy(value);
} else
panic(0, "mal-formed entry in conf file at line %d!", i+1);
}
gwlist_destroy(lines, (gwlist_item_destructor_t *)octstr_destroy);
octstr_destroy(sf);
/* Now check if config-source is set, use that. */
if ((grp = mms_cfg_get_single(cfg, octstr_imm("config-source"))) != NULL) {
Octstr *init = mms_cfg_get(cfg, grp, octstr_imm("config-library-init-param"));
cfg->cfg_funcs = _mms_load_module(cfg, grp, "config-library", "cfg_funcs", NULL);
if (cfg->cfg_funcs == NULL ||
cfg->cfg_funcs->read == NULL ||
(cfg->xcfg = cfg->cfg_funcs->read(init)) == NULL) {
mms_error(0, "mms_cfg", NULL, "Failed to load cfg reader library from conf!");
mms_cfg_destroy(cfg);
cfg = NULL;
}
octstr_destroy(init);
}
return cfg;
}
static int
combine_files (enum comb_command_type command,
struct lexer *lexer, struct dataset *ds)
{
struct comb_proc proc;
bool saw_by = false;
bool saw_sort = false;
struct casereader *active_file = NULL;
char *first_name = NULL;
char *last_name = NULL;
struct taint *taint = NULL;
size_t n_tables = 0;
size_t allocated_files = 0;
size_t i;
proc.files = NULL;
proc.n_files = 0;
proc.dict = dict_create (get_default_encoding ());
proc.output = NULL;
proc.matcher = NULL;
subcase_init_empty (&proc.by_vars);
proc.first = NULL;
proc.last = NULL;
proc.buffered_case = NULL;
proc.prev_BY = NULL;
dict_set_case_limit (proc.dict, dict_get_case_limit (dataset_dict (ds)));
lex_match (lexer, T_SLASH);
for (;;)
{
struct comb_file *file;
enum comb_file_type type;
if (lex_match_id (lexer, "FILE"))
type = COMB_FILE;
else if (command == COMB_MATCH && lex_match_id (lexer, "TABLE"))
{
type = COMB_TABLE;
n_tables++;
}
else
break;
lex_match (lexer, T_EQUALS);
if (proc.n_files >= allocated_files)
proc.files = x2nrealloc (proc.files, &allocated_files,
sizeof *proc.files);
file = &proc.files[proc.n_files++];
file->type = type;
subcase_init_empty (&file->by_vars);
subcase_init_empty (&file->src);
subcase_init_empty (&file->dst);
file->mv = NULL;
file->handle = NULL;
file->dict = NULL;
file->reader = NULL;
file->data = NULL;
file->is_sorted = true;
file->in_name = NULL;
file->in_var = NULL;
if (lex_match (lexer, T_ASTERISK))
{
if (!dataset_has_source (ds))
{
msg (SE, _("Cannot specify the active dataset since none "
"has been defined."));
goto error;
}
if (proc_make_temporary_transformations_permanent (ds))
msg (SE, _("This command may not be used after TEMPORARY when "
"the active dataset is an input source. "
"Temporary transformations will be made permanent."));
file->dict = dict_clone (dataset_dict (ds));
}
else
{
file->handle = fh_parse (lexer, FH_REF_FILE, dataset_session (ds));
if (file->handle == NULL)
goto error;
file->reader = any_reader_open (file->handle, NULL, &file->dict);
if (file->reader == NULL)
goto error;
}
while (lex_match (lexer, T_SLASH))
if (lex_match_id (lexer, "RENAME"))
{
if (!parse_dict_rename (lexer, file->dict))
goto error;
}
else if (lex_match_id (lexer, "IN"))
{
lex_match (lexer, T_EQUALS);
if (lex_token (lexer) != T_ID)
{
lex_error (lexer, NULL);
goto error;
}
if (file->in_name)
{
msg (SE, _("Multiple IN subcommands for a single FILE or "
"TABLE."));
goto error;
}
file->in_name = xstrdup (lex_tokcstr (lexer));
lex_get (lexer);
}
else if (lex_match_id (lexer, "SORT"))
{
file->is_sorted = false;
saw_sort = true;
}
if (!merge_dictionary (proc.dict, file))
goto error;
}
while (lex_token (lexer) != T_ENDCMD)
{
if (lex_match (lexer, T_BY))
{
const struct variable **by_vars;
size_t i;
bool ok;
if (saw_by)
{
lex_sbc_only_once ("BY");
goto error;
}
saw_by = true;
lex_match (lexer, T_EQUALS);
if (!parse_sort_criteria (lexer, proc.dict, &proc.by_vars,
&by_vars, NULL))
goto error;
ok = true;
for (i = 0; i < proc.n_files; i++)
{
struct comb_file *file = &proc.files[i];
size_t j;
for (j = 0; j < subcase_get_n_fields (&proc.by_vars); j++)
{
const char *name = var_get_name (by_vars[j]);
struct variable *var = dict_lookup_var (file->dict, name);
if (var != NULL)
subcase_add_var (&file->by_vars, var,
subcase_get_direction (&proc.by_vars, j));
else
{
if (file->handle != NULL)
msg (SE, _("File %s lacks BY variable %s."),
fh_get_name (file->handle), name);
else
msg (SE, _("Active dataset lacks BY variable %s."),
name);
ok = false;
}
}
assert (!ok || subcase_conformable (&file->by_vars,
&proc.files[0].by_vars));
}
free (by_vars);
if (!ok)
goto error;
}
else if (command != COMB_UPDATE && lex_match_id (lexer, "FIRST"))
{
if (first_name != NULL)
{
lex_sbc_only_once ("FIRST");
goto error;
}
lex_match (lexer, T_EQUALS);
if (!lex_force_id (lexer))
goto error;
first_name = xstrdup (lex_tokcstr (lexer));
lex_get (lexer);
}
else if (command != COMB_UPDATE && lex_match_id (lexer, "LAST"))
{
if (last_name != NULL)
{
lex_sbc_only_once ("LAST");
goto error;
}
lex_match (lexer, T_EQUALS);
if (!lex_force_id (lexer))
goto error;
last_name = xstrdup (lex_tokcstr (lexer));
lex_get (lexer);
}
else if (lex_match_id (lexer, "MAP"))
{
/* FIXME. */
}
else if (lex_match_id (lexer, "DROP"))
{
if (!parse_dict_drop (lexer, proc.dict))
goto error;
}
else if (lex_match_id (lexer, "KEEP"))
{
if (!parse_dict_keep (lexer, proc.dict))
goto error;
}
else
{
lex_error (lexer, NULL);
goto error;
}
if (!lex_match (lexer, T_SLASH) && lex_token (lexer) != T_ENDCMD)
{
lex_end_of_command (lexer);
goto error;
}
}
if (!saw_by)
{
if (command == COMB_UPDATE)
{
lex_sbc_missing ("BY");
goto error;
}
if (n_tables)
{
msg (SE, _("BY is required when %s is specified."), "TABLE");
goto error;
}
if (saw_sort)
{
msg (SE, _("BY is required when %s is specified."), "SORT");
goto error;
}
}
/* Add IN, FIRST, and LAST variables to master dictionary. */
for (i = 0; i < proc.n_files; i++)
{
struct comb_file *file = &proc.files[i];
if (!create_flag_var ("IN", file->in_name, proc.dict, &file->in_var))
goto error;
}
if (!create_flag_var ("FIRST", first_name, proc.dict, &proc.first)
|| !create_flag_var ("LAST", last_name, proc.dict, &proc.last))
goto error;
dict_delete_scratch_vars (proc.dict);
dict_compact_values (proc.dict);
/* Set up mapping from each file's variables to master
variables. */
for (i = 0; i < proc.n_files; i++)
{
struct comb_file *file = &proc.files[i];
size_t src_var_cnt = dict_get_var_cnt (file->dict);
size_t j;
file->mv = xnmalloc (src_var_cnt, sizeof *file->mv);
for (j = 0; j < src_var_cnt; j++)
{
struct variable *src_var = dict_get_var (file->dict, j);
struct variable *dst_var = dict_lookup_var (proc.dict,
var_get_name (src_var));
if (dst_var != NULL)
{
size_t n = subcase_get_n_fields (&file->src);
file->mv[n] = var_get_missing_values (src_var);
subcase_add_var (&file->src, src_var, SC_ASCEND);
subcase_add_var (&file->dst, dst_var, SC_ASCEND);
}
}
}
proc.output = autopaging_writer_create (dict_get_proto (proc.dict));
taint = taint_clone (casewriter_get_taint (proc.output));
/* Set up case matcher. */
proc.matcher = case_matcher_create ();
for (i = 0; i < proc.n_files; i++)
{
struct comb_file *file = &proc.files[i];
if (file->reader == NULL)
{
if (active_file == NULL)
{
proc_discard_output (ds);
file->reader = active_file = proc_open_filtering (ds, false);
}
else
file->reader = casereader_clone (active_file);
}
if (!file->is_sorted)
file->reader = sort_execute (file->reader, &file->by_vars);
taint_propagate (casereader_get_taint (file->reader), taint);
file->data = casereader_read (file->reader);
if (file->type == COMB_FILE)
case_matcher_add_input (proc.matcher, &file->by_vars,
&file->data, &file->is_minimal);
}
if (command == COMB_ADD)
execute_add_files (&proc);
else if (command == COMB_MATCH)
execute_match_files (&proc);
else if (command == COMB_UPDATE)
execute_update (&proc);
else
NOT_REACHED ();
case_matcher_destroy (proc.matcher);
proc.matcher = NULL;
close_all_comb_files (&proc);
if (active_file != NULL)
proc_commit (ds);
dataset_set_dict (ds, proc.dict);
dataset_set_source (ds, casewriter_make_reader (proc.output));
proc.dict = NULL;
proc.output = NULL;
free_comb_proc (&proc);
free (first_name);
free (last_name);
return taint_destroy (taint) ? CMD_SUCCESS : CMD_CASCADING_FAILURE;
error:
if (active_file != NULL)
proc_commit (ds);
free_comb_proc (&proc);
taint_destroy (taint);
free (first_name);
free (last_name);
return CMD_CASCADING_FAILURE;
}
/*************************************************************************
* dict_import: read in an ASCII dictionary.
*
* dict_fname - name of dictionary file
* parameters to create a DICTIONARY structure (see dict_create)
*
* Returns: pointer to created DICTIONARY structure
* (NULL on failure)
*
*************************************************************************/
DICTIONARY *dict_import( const char *dict_fname ,
const long initial_string_count ,
const long initial_hash_entries ,
const long max_chain_length )
{
DICTIONARY *dict;
char buffer[BUFLEN], ch;
int index, c, c0;
long number;
FILE *fi = NULL;
/***********
** Dictionary setup.
***********/
dict = dict_create( 4,
initial_string_count ,
initial_hash_entries ,
max_chain_length );
if ( dict == NULL )
goto err_exit;
/***********
** Read the dictionary file
** Each line should have one word or a string delimited by '|'
***********/
if ( (fi=fopen(dict_fname,"r")) == NULL )
goto err_exit;
while( fgets(buffer,BUFLEN,fi) != NULL ) {
c0 = 0;
/* Skip to non-blank */
while ( (c0<BUFLEN-2) && (buffer[c0]==' ') ) ++c0;
if ( buffer[c0] == '|' ) {
c = ++c0;
ch = '|';
} else {
c = c0;
ch = ' ';
} /* endif */
/* Scan to blank or matching '|' */
while ( (c<BUFLEN-1) && (buffer[c]!='\0') &&
(buffer[c]!='\n') && (buffer[c]!=ch) )
++c;
buffer[c] = '\0';
/* Insert the word */
if ( dict_insert(dict,buffer+c0,1,0,NULL,&number) == NULL )
goto err_exit;
} /* endwhile */
/***********
** Fill in the dictionary parameter vector.
***********/
if ( dict_set_parm_values(dict) == FALSE )
goto err_exit;
/***********
** Update the table of contents for HASH STTB STAR
***********/
if ( (index=dict_toc_index(dict,"HASH")) == -1 )
goto err_exit;
dict->toc[index].size = dict->table_size * sizeof(long);
if ( (index=dict_toc_index(dict,"STTB")) == -1 )
goto err_exit;
dict->toc[index].size = dict->string_max * sizeof(char);
if ( (index=dict_toc_index(dict,"STAR")) == -1 )
goto err_exit;
dict->toc[index].size = dict->array_size * sizeof(STRING_ENTRY);
/* Success. Return a pointer to the new dictionary. */
fclose(fi);
return( dict );
/* Failure. Ignominiously erase our tracks and return NULL. */
err_exit:
if ( fi != NULL )
fclose(fi);
dict_destroy( dict );
return NULL;
}
dict_t *
minerva_dict_create(void)
{
return dict_create();
}
int
main(int argc, char **argv) {
char *merylCount = 0L;
char *fastaName = 0L;
int arg=1;
while (arg < argc) {
if (strcmp(argv[arg], "-m") == 0) {
merylCount = argv[++arg];
} else if (strcmp(argv[arg], "-f") == 0) {
fastaName = argv[++arg];
} else {
fprintf(stderr, "unknown option '%s'\n", argv[arg]);
}
arg++;
}
if ((merylCount == 0L) || (fastaName == 0L)) {
fprintf(stderr, "usage: %s -m <meryl-name-prefix> -f <fasta-file>\n", argv[0]);
exit(1);
}
// Open the count files
//
merylStreamReader *MSR = new merylStreamReader(merylCount);
fprintf(stderr, "Mers are "uint32FMT" bases.\n", MSR->merSize());
fprintf(stderr, "There are "uint64FMT" unique (copy = 1) mers.\n", MSR->numberOfUniqueMers());
fprintf(stderr, "There are "uint64FMT" distinct mers.\n", MSR->numberOfDistinctMers());
fprintf(stderr, "There are "uint64FMT" mers total.\n", MSR->numberOfTotalMers());
// Guess how many mers we can fit into 512MB, then report how many chunks we need to do.
uint32 merSize = MSR->merSize();
uint64 memoryLimit = 700 * 1024 * 1024;
uint64 perMer = sizeof(kMerLite) + sizeof(dnode_t);
uint64 mersPerBatch = memoryLimit / perMer;
uint32 numBatches = MSR->numberOfDistinctMers() / mersPerBatch;
uint32 batch = 0;
dnode_t *nodes = new dnode_t [mersPerBatch];
kMerLite *mers = new kMerLite [mersPerBatch];
if (MSR->numberOfDistinctMers() % mersPerBatch)
numBatches++;
fprintf(stderr, "perMer: "uint64FMT" bytes ("uint64FMT" for kMerLite, "uint64FMT" for dnode_t.\n",
perMer, (uint64)sizeof(kMerLite), (uint64)sizeof(dnode_t));
fprintf(stderr, "We can fit "uint64FMT" mers into "uint64FMT"MB.\n", mersPerBatch, memoryLimit >> 20);
fprintf(stderr, "So we need "uint32FMT" batches to verify the count.\n", numBatches);
while (MSR->validMer()) {
uint64 mersRemain = mersPerBatch;
dict_t *merDict = dict_create(mersPerBatch, kMerLiteSort);
batch++;
// STEP 1: Insert mersPerBatch into the merDict
//
fprintf(stderr, "STEP 1 BATCH "uint32FMTW(2)": Insert into merDict\n", batch);
while (MSR->nextMer() && mersRemain) {
mersRemain--;
mers[mersRemain] = MSR->theFMer();
// initialize the node with the value, then insert the node
// into the tree using the key
int32 val = (int32)MSR->theCount();
dnode_init(&nodes[mersRemain], (void *)val);
dict_insert(merDict, &nodes[mersRemain], &mers[mersRemain]);
}
// STEP 2: Stream the original file, decrementing the count
//
fprintf(stderr, "STEP 2 BATCH "uint32FMTW(2)": Stream fasta\n", batch);
seqStream *CS = new seqStream(fastaName, true);
merStream *MS = new merStream(new kMerBuilder(merSize), CS);
kMerLite mer;
dnode_t *nod;
while (MS->nextMer()) {
mer = MS->theFMer();
nod = dict_lookup(merDict, &mer);
if (nod != 0L) {
int32 val = (int32)dnode_get(nod);
val--;
dnode_put(nod, (void *)val);
} else {
// Unless the whole meryl file fit into our merDict, we cannot warn if
// we don't find mers.
//
if (numBatches == 1) {
char str[1024];
fprintf(stderr, "Didn't find node for mer '%s'\n", mer.merToString(merSize, str));
}
}
}
delete MS;
delete CS;
// STEP 3: Check every node in the tree to make sure that the counts
// are exactly zero.
//
fprintf(stderr, "STEP 3 BATCH "uint32FMTW(2)": Check\n", batch);
nod = dict_first(merDict);
while (nod) {
int32 val = (int32)dnode_get(nod);
kMerLite const *nodmer = (kMerLite const *)dnode_getkey(nod);
if (val != 0) {
char str[1024];
fprintf(stderr, "Got count "int32FMT" for mer '%s'\n",
val,
nodmer->merToString(merSize, str));
}
nod = dict_next(merDict, nod);
}
// STEP 4: Destroy the dictionary.
//
fprintf(stderr, "STEP 4 BATCH "uint32FMTW(2)": Destroy\n", batch);
while ((nod = dict_first(merDict)))
dict_delete(merDict, nod);
dict_destroy(merDict);
}
}
int
main(int argc, char **argv) {
uint32 pNum = 0;
uint32 pAlloc = 8388608;
uint32 estID = ~uint32ZERO;
bool *found = 0L;
// From fixPolishesIID.c
IIDdict = 0L;
SEQdict = 0L;
GENdict = 0L;
// Incorporated from sortPolishes
mergeFilesLen = 0;
mergeFilesMax = sysconf(_SC_OPEN_MAX);
mergeFiles = new FILE * [mergeFilesMax];
mergeNames = new char * [mergeFilesMax];
mergePolishes = new sim4polish * [mergeFilesMax];
// Default to printing stats on stdout.
sFile = stdout;
int arg = 1;
while (arg < argc) {
if (strcmp(argv[arg], "-n") == 0) {
pAlloc = strtouint32(argv[++arg], 0L);
} else if (strcmp(argv[arg], "-fpart") == 0) {
arg++;
fprintf(stderr, "reading query deflines from '%s'\n", argv[arg]);
IIDdict = dict_create(DICTCOUNT_T_MAX, headerCompare);
addToDict(IIDdict, argv[arg]);
} else if (strcmp(argv[arg], "-g") == 0) {
++arg;
fprintf(stderr, "reading genomic deflines from '%s'\n", argv[arg]);
GENdict = dict_create(DICTCOUNT_T_MAX, headerCompare);
addToDict(GENdict, argv[arg]);
} else if (strcmp(argv[arg], "-F") == 0) {
++arg;
fprintf(stderr, "reading query deflines from '%s'\n", argv[arg]);
SEQdict = dict_create(DICTCOUNT_T_MAX, headerCompare);
addToDict(SEQdict, argv[arg]);
} else if (strcmp(argv[arg], "-f") == 0) {
++arg;
SEQ = new seqCache(argv[arg]);
} else if (strcmp(argv[arg], "-q") == 0) {
++arg;
QLT = new seqCache(argv[arg]);
} else if (strcmp(argv[arg], "-filter") == 0) {
filter = atof(argv[++arg]);
doFiltering = true;
} else if (strcmp(argv[arg], "-output") == 0) {
char cmd[1024] = {0};
errno = 0;
++arg;
if (strcmp(argv[arg] + strlen(argv[arg]) - 4, ".bz2") == 0) {
sprintf(cmd, "bzip2 -1c > %s", argv[arg]);
oFile = popen(cmd, "w");
oFileIsPipe = 1;
} else if (strcmp(argv[arg] + strlen(argv[arg]) - 3, ".gz") == 0) {
sprintf(cmd, "gzip -1c > %s", argv[arg]);
oFile = popen(cmd, "w");
oFileIsPipe = 1;
} else {
fprintf(stderr, "Got %s, not .bz2 not .gz!\n", argv[arg]);
exit(1);
}
if (errno)
fprintf(stderr, "Failed to open '%s': %s\n", cmd, strerror(errno));
doFiltering = true;
} else if (strcmp(argv[arg], "-scores") == 0) {
errno = 0;
sFile = fopen(argv[++arg], "w");
if (errno)
fprintf(stderr, "Failed to open '%s': %s\n", argv[arg-1], strerror(errno));
doFiltering = true;
} else if (strcmp(argv[arg], "-unique") == 0) {
char cmd[1024] = {0};
errno = 0;
arg++;
if (strcmp(argv[arg] + strlen(argv[arg]) - 4, ".bz2") == 0)
sprintf(cmd, "bzip2 -1c > %s", argv[arg]);
else if (strcmp(argv[arg] + strlen(argv[arg]) - 3, ".gz") == 0)
sprintf(cmd, "gzip -1c > %s", argv[arg]);
else
sprintf(cmd, "cat > %s", argv[arg]);
uFile = popen(cmd, "w");
if (errno)
fprintf(stderr, "Failed to open '%s': %s\n", cmd, strerror(errno));
doFiltering = true;
} else if (strncmp(argv[arg], "-M", 2) == 0) {
arg++;
while ((arg < argc) && (fileExists(argv[arg]))) {
if (mergeFilesLen >= mergeFilesMax) {
fprintf(stderr, "%s: ERROR! Too many input files! Should be less than %d\n", argv[0], mergeFilesMax);
exit(1);
}
mergeNames[mergeFilesLen] = argv[arg];
mergeFiles[mergeFilesLen++] = openFile(argv[arg], "r");
arg++;
}
arg--;
} else {
fprintf(stderr, "unknown option: %s\n", argv[arg]);
}
arg++;
}
if (doFiltering) {
if (uFile == 0L)
fprintf(stderr, "ERROR: -unique is required\n"), exit(1);
if (sFile == 0L)
fprintf(stderr, "ERROR: -scores is required\n"), exit(1);
if ((filter < 0.0) || (filter > 1.0))
fprintf(stderr, "ERROR: -filter value of %f invalid. 0 <= F <= 100.\n", filter), exit(1);
}
if ((IIDdict == 0L) || (SEQdict == 0L) || (GENdict == 0L)) {
fprintf(stderr, "WARNING! No sequence dictionaries, NOT FIXING IIDs! (supply -fpart, -f and -g)\n");
}
if ((SEQ == 0L) || (QLT == 0L)) {
fprintf(stderr, "I need -f and -q\n");
exit(1);
}
// We no longer require that input polishes be sorted increasingly;
// now they only must be grouped. This remembers if we've seen a
// match or not. At the end, we'll analyze() those we haven't done
// already.
//
found = new bool [ SEQ->getNumberOfSequences() ];
for (uint32 i=0; i<SEQ->getNumberOfSequences(); i++)
found[i] = false;
// Initialize the merge -- if no merge files, nothing done!
//
for (int i=0; i<mergeFilesLen; i++) {
mergePolishes[i] = new sim4polish(mergeFiles[i]);
fixIID(mergePolishes[i], IIDdict);
}
// Read polishes, picking the best when we see a change in the
// estID.
sim4polish **p = new sim4polish * [pAlloc];
sim4polish *q;
while ((q = nextPolish()) != 0L) {
if ((q->_estID != estID) && (pNum > 0)) {
//fprintf(stderr, "PickBest for estID "uint32FMT"\n", estID);
found[estID] = true;
pickBest(p, pNum);
pNum = 0;
}
if (pNum >= pAlloc) {
sim4polish **P = new sim4polish * [pAlloc * 2];
memcpy(p, P, sizeof(sim4polish *) * pAlloc);
delete [] p;
p = P;
pAlloc *= 2;
}
p[pNum++] = q;
estID = q->_estID;
}
if (pNum > 0) {
found[estID] = true;
pickBest(p, pNum);
}
// Attempt cleanup
//
for (int i=0; i<mergeFilesLen; i++)
closeFile(mergeFiles[i], mergeNames[i]);
for (estID=0; estID < SEQ->getNumberOfSequences(); estID++)
if (found[estID] == false)
analyze(estID, 0, SEQ->getSequenceLength(estID), SEQ->getSequenceLength(estID), true, 'M');
delete [] mergeFiles;
delete [] mergeNames;
delete [] mergePolishes;
if (oFile) pclose(oFile);
if (uFile) pclose(uFile);
if (sFile) fclose(sFile);
fprintf(stderr, "Uni:"uint32FMTW(8)" Con:"uint32FMTW(8)" (T:"uint32FMTW(8)" M:"uint32FMTW(8)" I:"uint32FMTW(8)" S:"uint32FMTW(8)" N:"uint32FMTW(8)") Inc:"uint32FMTW(8)" -- Save:"uint32FMTW(8)" Lost:"uint32FMTW(8)"\n",
statOneMatch,
statConsistent, consistentTie, consistentMatches, consistentIdentity, consistentTooShort, consistentNot,
statInconsistent,
statUnique, statLost);
fprintf(stderr, "total: LQ:"uint32FMT" MQ:"uint32FMT" RQ:"uint32FMT"\n",
totLQ, totMQ, totRQ);
return(0);
}
/// Convert all raw PAs in the group into a single set of "cooked" PAs
/// under the leader.
/// @param[in] ca the object pointer
void
ca_coalesce(ca_o ca)
{
dict_t *dict_raw, *dict_cooked;
dnode_t *dnpr, *dnpc, *next;
dict_raw = ca->ca_raw_pa_dict;
assert(!ca->ca_cooked_pa_dict);
if ((dict_cooked = dict_create(DICTCOUNT_T_MAX, pa_cmp_by_pathname))) {
ca->ca_cooked_pa_dict = dict_cooked;
} else {
putil_syserr(2, "dict_create()");
}
for (dnpr = dict_first(dict_raw); dnpr;) {
pa_o raw_pa, ckd_pa;
next = dict_next(dict_raw, dnpr);
raw_pa = (pa_o)dnode_getkey(dnpr);
_ca_verbosity_pa(raw_pa, ca, "COALESCING");
// All data is in the key - that's why the value can be null.
if ((dnpc = dict_lookup(dict_cooked, raw_pa))) {
int keep_cooked = 0;
ckd_pa = (pa_o)dnode_getkey(dnpc);
if (!pa_is_read(raw_pa) && !pa_is_read(ckd_pa)) {
moment_s raw_timestamp, ckd_timestamp;
// If they're both destructive ops (non-read) then we
// need to consider timestamps and use the later one.
if (pa_has_timestamp(raw_pa) && pa_has_timestamp(ckd_pa)) {
// If the PA's have their own timestamps, use them.
raw_timestamp = pa_get_timestamp(raw_pa);
ckd_timestamp = pa_get_timestamp(ckd_pa);
} else {
// Otherwise key off the file times. This is for
// support of "dummy" PAs as used in shopping.
raw_timestamp = pa_get_moment(raw_pa);
ckd_timestamp = pa_get_moment(ckd_pa);
}
if (moment_cmp(raw_timestamp, ckd_timestamp, NULL) <= 0) {
// Cooked write op is newer and can stay.
keep_cooked = 1;
}
} else if (pa_is_read(raw_pa)) {
// There's no point replacing a read with another read,
// so regardless of whether the current cooked PA is a
// read or write, it can stay.
keep_cooked = 1;
} else {
// A write always beats a read.
}
if (!keep_cooked) {
dict_delete(dict_cooked, dnpc);
dnode_destroy(dnpc);
_ca_verbosity_pa(ckd_pa, ca, "REMOVING");
pa_destroy(ckd_pa);
if (!(dnpc = dnode_create(NULL))) {
putil_syserr(2, "dnode_create()");
}
ckd_pa = pa_copy(raw_pa);
dict_insert(dict_cooked, dnpc, ckd_pa);
}
} else {
if (!(dnpc = dnode_create(NULL))) {
putil_syserr(2, "dnode_create()");
}
ckd_pa = pa_copy(raw_pa);
dict_insert(dict_cooked, dnpc, ckd_pa);
}
// Clean up the raw set as we move PAs to the cooked one.
dict_delete(dict_raw, dnpr);
dnode_destroy(dnpr);
dnpr = next;
}
return;
}
/* Parses and executes the AGGREGATE procedure. */
int
cmd_aggregate (struct lexer *lexer, struct dataset *ds)
{
struct dictionary *dict = dataset_dict (ds);
struct agr_proc agr;
struct file_handle *out_file = NULL;
struct casereader *input = NULL, *group;
struct casegrouper *grouper;
struct casewriter *output = NULL;
bool copy_documents = false;
bool presorted = false;
bool saw_direction;
bool ok;
memset(&agr, 0 , sizeof (agr));
agr.missing = ITEMWISE;
agr.src_dict = dict;
subcase_init_empty (&agr.sort);
/* OUTFILE subcommand must be first. */
lex_match (lexer, T_SLASH);
if (!lex_force_match_id (lexer, "OUTFILE"))
goto error;
lex_match (lexer, T_EQUALS);
if (!lex_match (lexer, T_ASTERISK))
{
out_file = fh_parse (lexer, FH_REF_FILE, dataset_session (ds));
if (out_file == NULL)
goto error;
}
if (out_file == NULL && lex_match_id (lexer, "MODE"))
{
lex_match (lexer, T_EQUALS);
if (lex_match_id (lexer, "ADDVARIABLES"))
{
agr.add_variables = true;
/* presorted is assumed in ADDVARIABLES mode */
presorted = true;
}
else if (lex_match_id (lexer, "REPLACE"))
{
agr.add_variables = false;
}
else
goto error;
}
if ( agr.add_variables )
agr.dict = dict_clone (dict);
else
agr.dict = dict_create (dict_get_encoding (dict));
dict_set_label (agr.dict, dict_get_label (dict));
dict_set_documents (agr.dict, dict_get_documents (dict));
/* Read most of the subcommands. */
for (;;)
{
lex_match (lexer, T_SLASH);
if (lex_match_id (lexer, "MISSING"))
{
lex_match (lexer, T_EQUALS);
if (!lex_match_id (lexer, "COLUMNWISE"))
{
lex_error_expecting (lexer, "COLUMNWISE", NULL);
goto error;
}
agr.missing = COLUMNWISE;
}
else if (lex_match_id (lexer, "DOCUMENT"))
copy_documents = true;
else if (lex_match_id (lexer, "PRESORTED"))
presorted = true;
else if (lex_force_match_id (lexer, "BREAK"))
{
int i;
lex_match (lexer, T_EQUALS);
if (!parse_sort_criteria (lexer, dict, &agr.sort, &agr.break_vars,
&saw_direction))
goto error;
agr.break_var_cnt = subcase_get_n_fields (&agr.sort);
if (! agr.add_variables)
for (i = 0; i < agr.break_var_cnt; i++)
dict_clone_var_assert (agr.dict, agr.break_vars[i]);
/* BREAK must follow the options. */
break;
}
else
goto error;
}
if (presorted && saw_direction)
msg (SW, _("When PRESORTED is specified, specifying sorting directions "
"with (A) or (D) has no effect. Output data will be sorted "
"the same way as the input data."));
/* Read in the aggregate functions. */
lex_match (lexer, T_SLASH);
if (!parse_aggregate_functions (lexer, dict, &agr))
goto error;
/* Delete documents. */
if (!copy_documents)
dict_clear_documents (agr.dict);
/* Cancel SPLIT FILE. */
dict_set_split_vars (agr.dict, NULL, 0);
/* Initialize. */
agr.case_cnt = 0;
if (out_file == NULL)
{
/* The active dataset will be replaced by the aggregated data,
so TEMPORARY is moot. */
proc_cancel_temporary_transformations (ds);
proc_discard_output (ds);
output = autopaging_writer_create (dict_get_proto (agr.dict));
}
else
{
output = any_writer_open (out_file, agr.dict);
if (output == NULL)
goto error;
}
input = proc_open (ds);
if (!subcase_is_empty (&agr.sort) && !presorted)
{
input = sort_execute (input, &agr.sort);
subcase_clear (&agr.sort);
}
for (grouper = casegrouper_create_vars (input, agr.break_vars,
agr.break_var_cnt);
casegrouper_get_next_group (grouper, &group);
casereader_destroy (group))
{
struct casereader *placeholder = NULL;
struct ccase *c = casereader_peek (group, 0);
if (c == NULL)
{
casereader_destroy (group);
continue;
}
initialize_aggregate_info (&agr);
if ( agr.add_variables )
placeholder = casereader_clone (group);
{
struct ccase *cg;
for (; (cg = casereader_read (group)) != NULL; case_unref (cg))
accumulate_aggregate_info (&agr, cg);
}
if (agr.add_variables)
{
struct ccase *cg;
for (; (cg = casereader_read (placeholder)) != NULL; case_unref (cg))
dump_aggregate_info (&agr, output, cg);
casereader_destroy (placeholder);
}
else
{
dump_aggregate_info (&agr, output, c);
}
case_unref (c);
}
if (!casegrouper_destroy (grouper))
goto error;
if (!proc_commit (ds))
{
input = NULL;
goto error;
}
input = NULL;
if (out_file == NULL)
{
struct casereader *next_input = casewriter_make_reader (output);
if (next_input == NULL)
goto error;
dataset_set_dict (ds, agr.dict);
dataset_set_source (ds, next_input);
agr.dict = NULL;
}
else
{
ok = casewriter_destroy (output);
output = NULL;
if (!ok)
goto error;
}
agr_destroy (&agr);
fh_unref (out_file);
return CMD_SUCCESS;
error:
if (input != NULL)
proc_commit (ds);
casewriter_destroy (output);
agr_destroy (&agr);
fh_unref (out_file);
return CMD_CASCADING_FAILURE;
}
static void mms_queue_run(char *dir,
int (*deliver)(MmsEnvelope *),
double sleepsecs, int num_threads, int *rstop)
{
int i, qstop = 0;
List *stack = gwlist_create();
// static struct Qthread_t *tlist;
struct Qthread_t *tlist;
debug("", 0, "mms_queue_run: %s", dir);
max_live_exceeded_warning_issued = 0;
number_of_threads = num_threads;
gw_assert(num_threads>0);
if (tlists == NULL)
tlists = dict_create(10, NULL);
Octstr *odir = octstr_create(dir);
tlist = gw_malloc(num_threads*sizeof tlist[0]);
dict_put(tlists, odir, tlist);
gw_assert(tlist == dict_get(tlists,odir));
debug("",0,"tlist allocated at addr %p", &tlist[0]);
octstr_destroy(odir);
for (i = 0; i<num_threads; i++) { /* Create threads for sending. */
debug("",0,"%s tlist[%d] has addr: %p", dir, i, &tlist[i]);
tlist[i].l = gwlist_create();
gwlist_add_producer(tlist[i].l);
tlist[i].deliver = deliver;
gwthread_create((gwthread_func_t *)tdeliver, &tlist[i]);
}
i = 0; /* For stepping through above array. */
do {
Octstr *xdir = NULL;
gwlist_append(stack, octstr_create("")); /* Put initial dir on there. */
while (!*rstop && (xdir = gwlist_extract_first(stack)) != NULL) {
int ret = run_dir(dir, octstr_get_cstr(xdir), tlist, num_threads, &i, stack);
octstr_destroy(xdir);
xdir = NULL;
if (ret < 0) {
if (ret <= -2)
qstop = 1;
goto qloop;
}
}
octstr_destroy(xdir);
if (*rstop)
break;
qloop:
gwthread_sleep(sleepsecs);
} while (!qstop);
/* We are out of the queue, time to go away. */
for (i = 0; i<num_threads; i++)
if (tlist[i].l)
gwlist_remove_producer(tlist[i].l);
gwthread_join_every((gwthread_func_t *)tdeliver); /* Wait for them all to terminate. */
for (i = 0; i<num_threads; i++)
if (tlist[i].l)
gwlist_destroy(tlist[i].l,NULL); /* Final destroy if needed. */
gw_free(tlist);
gwlist_destroy(stack, (gwlist_item_destructor_t *)octstr_destroy);
return;
}
int main(int argc, char **args){
if(argc != 3){
fprintf(stderr,"%s RECV SEND\n",args[0]);
exit(1);
}
signal(SIGINT,&call_for_stop);
bstring pull_addr = bfromcstr(args[1]);
bstring pub_addr = bfromcstr(args[2]);
mongrel2_ctx *ctx = mongrel2_init(1); // Yes for threads?
mongrel2_socket *pull_socket = mongrel2_pull_socket(ctx);
mongrel2_connect(pull_socket, bdata(pull_addr));
pub_socket = mongrel2_pub_socket(ctx);
mongrel2_connect(pub_socket, bdata(pub_addr));
mongrel2_set_identity(pub_socket, bdata(&SENDER) );
mongrel2_request *request;
// Polling is done to show how to do a clean shutdown
int poll_response;
zmq_pollitem_t socket_tracker;
socket_tracker.socket = pull_socket->zmq_socket;
socket_tracker.events = ZMQ_POLLIN;
// Let's try out some ADT goodness
dict_t* dict = dict_create(DICTCOUNT_T_MAX, compare_session);
dict_set_allocator(dict, alloc_dict, free_dict, NULL);
dnode_t* tempnode = NULL;
m2_ws_session_data *counter = NULL;
int retval = 0;
while(shutdown != 1){
poll_response = zmq_poll(&socket_tracker,1,500*1000);
if(poll_response > 0){
request = mongrel2_recv(pull_socket);
fprintf(stdout,"got something...\n");
if(request != NULL && mongrel2_request_for_disconnect(request) != 1){
m2_ws_session_id* incoming = calloc(1,sizeof(m2_ws_session_id));
incoming->req = request;
printf("Looking at incoming->conn_id = %d\n",incoming->req->conn_id);
tempnode = dict_lookup(dict,incoming);
if(tempnode == NULL){
mongrel2_ws_reply_upgrade(request,pub_socket);
counter = calloc(1,sizeof(m2_ws_session_data));
counter->times_seen = 0;
retval = dict_alloc_insert(dict,incoming,counter);
assert(retval == 1);
} else {
free(incoming);
counter = dnode_get(tempnode);
counter->times_seen += 1;
}
if(blength(request->body) > 0){
if(tempnode && mongrel2_ws_frame_get_fin(blength(request->body),
(uint8_t*)bdata(request->body))){
printf("Hey, it's a close\n");
dict_delete_free(dict,tempnode);
mongrel2_disconnect(pub_socket,request);
}
} else {
bstring randmsg = genrandmsg();
mongrel2_ws_reply(pub_socket,request,randmsg);
}
printf("FYI: we've got %ld entries\n",dict_count(dict));
} else {
fprintf(stdout,"Connection %d disconnected\n", request->conn_id);
}
} else if (poll_response < 0){
fprintf(stdout, "Error on poll!");
shutdown = 1;
}
}
// bstring msg = bformat("{\"msg\" : \"hi there %d\"}", request->conn_id);
// fprintf(stdout,"Sending new msg: '%*s'",blength(msg),bdata(msg));
// mongrel2_ws_reply(pub_socket,request,msg);
// bdestroy(msg);
// mongrel2_request_finalize(request);
// mongrel2_reply(pub_socket,request,bfromcstr(""));
bdestroy(pull_addr);
bdestroy(pub_addr);
mongrel2_close(pull_socket);
mongrel2_close(pub_socket);
mongrel2_deinit(ctx);
fprintf(stdout,"\nClean shutdown done! Thanks for playing!\n");
return 0;
}
int main() {
size_t i = 0, m, max=100000;
char buf[128];
struct timeval beg, end;
long sec ;
dict *d = dict_create(&opts);
for(m = 0; m < 3; m++) {
for(i = 0; i < max; i++) {
memset(buf, 0, sizeof(buf));
snprintf(buf, sizeof(buf), ";;%lu;;", i);
dict_add(d, buf, buf);
}
printf("add %d, %d %llu\n", DICT_USED(d), DICT_CAP(d),used_mem());
for(i = 0; i < max; i++) {
dict_entry *entry;
memset(buf, 0, sizeof(buf));
snprintf(buf, sizeof(buf), ";;%lu;;", i);
entry = dict_find(d, buf);
if(strcmp((char*)entry->value, (char*)buf) != 0)
abort();
}
printf("find %d, %d %llu\n", DICT_USED(d), DICT_CAP(d),used_mem());
for(i = 0; i < max; i++) {
memset(buf, 0, sizeof(buf));
snprintf(buf, sizeof(buf), ";;%lu;;", i);
dict_replace(d, buf, buf);
}
printf("prelace %d, %d %llu\n", DICT_USED(d), DICT_CAP(d),used_mem());
for(i = 0; i < max; i++) {
memset(buf, 0, sizeof(buf));
snprintf(buf, sizeof(buf), ";;%lu;;", i);
dict_del(d, buf);
}
printf("delete %d, %d %llu\n", DICT_USED(d), DICT_CAP(d),used_mem());
}
dict_expand(d, max*2);
printf("%d, %d %llu\n", DICT_USED(d), DICT_CAP(d),used_mem());
printf("%d %llu\n", DICT_CAP(d),used_mem());
dict_destroy(d);
printf("%llu\n", used_mem());
d = dict_create(&opts);
gettimeofday(&beg, NULL);
for(i = 0; i < 1000000; i++) {
memset(buf, 0, sizeof(buf));
snprintf(buf, sizeof(buf), "%lu", i);
dict_add(d, buf, buf);
}
gettimeofday(&end, NULL);
sec = (end.tv_sec - beg.tv_sec)*1000;
sec += (end.tv_usec - beg.tv_usec)/1000;
printf("add msec %ld, mm: %llu\n", sec, used_mem());
gettimeofday(&beg, NULL);
dict_iterator *iter = dict_get_iterator(d);
dict_entry *entry = NULL;
i = 0;
while((entry = dict_iterator_next(iter)) != NULL){
i++;
}
dict_iterator_destroy(iter);
gettimeofday(&end, NULL);
sec = (end.tv_sec - beg.tv_sec)*1000;
sec += (end.tv_usec - beg.tv_usec)/1000;
printf("iterator msec %ld, times: %lu mm: %llu\n", sec, i, used_mem());
gettimeofday(&beg, NULL);
for(i = 0; i < 1000000; i++) {
memset(buf, 0, sizeof(buf));
snprintf(buf, sizeof(buf), "%lu", i);
dict_del(d, buf);
}
gettimeofday(&end, NULL);
sec = (end.tv_sec - beg.tv_sec)*1000;
sec += (end.tv_usec - beg.tv_usec)/1000;
printf("remove msec %ld, mm: %llu\n", sec, used_mem());
dict_destroy(d);
printf("%llu\r\n", used_mem());
return 0;
}
