int main(int argc, char **argv){
int c;
char *msg;
int colnames = 'y',
rownames = 0,
tab_exists_check = 0;
char **field_names = NULL;
Asprintf(&msg, "Usage: %s [-d delimiters] text_file table_name dbname\n"
"\n"
"If the input text file name is a single dash, -, then read from STDIN.\n"
"Input must be plain ASCII or UTF-8.\n"
" -d\t\tthe single-character delimiters to use, e.g., -d \" ,\" or -d \"\\t\" (which you \n"
" \t\t\twill almost certainly have to write as -d \"\\\\t\") (default: \"|,\\t\", meaning \n"
" \t\t\tthat any of a pipe, comma, or tab will delimit separate entries)\n"
" -nc\t\tdata does not include column names\n"
" -n regex\t\tcase-insensitive regular expression indicating Null values (default: NaN)\n"
" -m\t\tuse a MySQL database (default: SQLite)\n"
" -f\t\tfixed width field ends: -f\"3,8,12,17\" (first char is one, not zero)\n"
" -u\t\tmysql username\n"
" -p\t\tmysql password\n"
" -r\t\tdata includes row names\n"
" -v\t\tverbosity\n"
" -N\t\ta comma-separated list of column names: -N\"apple,banana,carrot,durian\"\n"
" -en\t\tif table exists, do nothing and exit\n"
" -ed\t\tif table exists, retain the table, delete all data, refill with the new data (i.e., call 'delete * from your_table')\n"
" -eo\t\tif table exists, overwrite the table from scratch (deleting the previous table entirely)\n"
" -ea\t\tif table exists, append new data to the existing table\n"
" -h\t\tdisplay this help and exit\n"
"\n"
, argv[0]);
int * field_list = NULL;
char if_exists = 'n';
if(argc<3){
printf("%s", msg);
return 0;
}
while ((c = getopt (argc, argv, "n:d:e:f:hmp:ru:vN:O")) != -1)
if (c=='n') {
if (optarg[0]=='c') colnames='n';
else apop_opts.nan_string = optarg;
}
else if (c=='N') {
apop_data *field_name_data;
apop_regex(optarg, " *([^,]*[^ ]) *(,|$) *", &field_name_data);
Apop_stopif(!field_name_data, return 1, 0, "'%s' should be a "
"comma-delimited list of field names, but I had trouble "
"parsing it as such.", optarg);
apop_data_transpose(field_name_data);
field_names = field_name_data->text[0];
}
/** Append one list of names to another.
If the first list is empty, then this is a copy function.
\param n1 The first set of names (no default, must not be \c NULL)
\param nadd The second set of names, which will be appended after the first. (no default. If \c NULL, a no-op.)
\param type1 Either 'c', 'r', 't', or 'v' stating whether you are merging the
columns, rows, text, or vector. If 'v', then ignore \c typeadd and just overwrite the
target vector name with the source name. (default: 'r')
\param typeadd Either 'c', 'r', 't', or 'v' stating whether you are merging the columns, rows, or text. If 'v', then overwrite the target with the source vector name. (default: type1)
*/
APOP_VAR_HEAD void apop_name_stack(apop_name * n1, apop_name *nadd, char type1, char typeadd){
apop_name * apop_varad_var(nadd, NULL);
if (!nadd) return;
apop_name * apop_varad_var(n1, NULL);
Apop_stopif(!n1, return, 0, "Can't stack onto a NULL set of names (which n1 is).");
char apop_varad_var(type1, 'r');
char apop_varad_var(typeadd, type1);
APOP_VAR_ENDHEAD
int i;
apop_name counts = (apop_name){.rowct=nadd->rowct, .textct = nadd->textct, .colct = nadd->colct};//Necessary when stacking onto self.;
if (typeadd == 'v')
apop_name_add(n1, nadd->vector, 'v');
else if (typeadd == 'r')
for (i=0; i< counts.rowct; i++)
apop_name_add(n1, nadd->row[i], type1);
else if (typeadd == 't')
for (i=0; i< counts.textct; i++)
apop_name_add(n1, nadd->text[i], type1);
else if (typeadd == 'c')
for (i=0; i< counts.colct; i++)
apop_name_add(n1, nadd->col[i], type1);
else Apop_notify(1, "'%c' sent to apop_name_stack, but the only "
"valid options are r t c v. Doing nothing.", typeadd);
}
/** Copy one \ref apop_name structure to another. That is, all data is duplicated.
Used internally by \ref apop_data_copy, but sometimes useful by itself. For example,
say that we have an \ref apop_data struct named \c d and a \ref gsl_matrix of the same
dimensions named \c m; we could give \c m the labels from \c d for printing:
\code
apop_data *wrapped = &(apop_data){.matrix=m, .names=apop_name_copy(d)};
apop_data_print(wrapped);
apop_name_free(wrapped->names); //wrapped itself is auto-allocated; do not free.
\endcode
\param in The input names
\return A \ref apop_name struct with copies of all input names.
*/
apop_name * apop_name_copy(apop_name *in){
apop_name *out = apop_name_alloc();
apop_name_stack(out, in, 'v');
apop_name_stack(out, in, 'c');
apop_name_stack(out, in, 'r');
apop_name_stack(out, in, 't');
Asprintf(&out->title, "%s", in->title);
return out;
}
static int
Exists(void *obj, const AG_Dbt *key)
{
AG_DbMySQL *db = obj;
char *ks, *q;
ks = EncodeKey(key);
Asprintf(&q, AG_GetStringP(db,"get-cmd"), ks);
Free(ks);
if (mysql_query(db->my, q) != 0) {
AG_SetError("Get: %s", mysql_error(db->my));
return (-1);
}
Free(q);
return (mysql_field_count(db->my) > 0) ? 1 : 0;
}
/** Adds a name to the \ref apop_name structure. Puts it at the end of the given list.
\param n An existing, allocated \ref apop_name structure.
\param add_me A string. If \c NULL, do nothing; return -1.
\param type 'r': add a row name<br>
'c': add a matrix column name<br>
't': add a text column name<br>
'h': add a title (i.e., a header).<br>
'v': add (or overwrite) the vector name<br>
\return Returns the number of rows/cols/depvars after you have added the new one. But if \c add_me is \c NULL, return -1.
*/
int apop_name_add(apop_name * n, char const *add_me, char type){
if (!add_me)
return -1;
if (type == 'h'){
free(n->title);
Asprintf(&n->title, "%s", add_me);
return 1;
}
if (type == 'v'){
n->vector = realloc(n->vector, strlen(add_me) + 1);
strcpy(n->vector, add_me);
return 1;
}
if (type == 'r'){
n->rowct++;
n->row = realloc(n->row, sizeof(char*) * n->rowct);
n->row[n->rowct -1] = malloc(strlen(add_me) + 1);
strcpy(n->row[n->rowct -1], add_me);
n->rowhash = realloc(n->rowhash, n->rowct * sizeof(unsigned long));
n->rowhash[n->rowct-1] = apop_name_hash(add_me);
return n->rowct;
}
if (type == 't'){
n->textct++;
n->text = realloc(n->text, sizeof(char*) * n->textct);
n->text[n->textct -1] = malloc(strlen(add_me) + 1);
strcpy(n->text[n->textct -1], add_me);
n->texthash = realloc(n->texthash, n->textct * sizeof(unsigned long));
n->texthash[n->textct-1] = apop_name_hash(add_me);
return n->textct;
}
//else assume (type == 'c')
Apop_stopif(type != 'c', /*keep going.*/,
2,"You gave me >%c<, I'm assuming you meant c; "
" copying column names.", type);
n->colct++;
n->col = realloc(n->col, sizeof(char*) * n->colct);
n->col[n->colct -1] = malloc(strlen(add_me) + 1);
strcpy(n->col[n->colct -1], add_me);
n->colhash = realloc(n->colhash, n->colct * sizeof(unsigned long));
n->colhash[n->colct-1] = apop_name_hash(add_me);
return n->colct;
}
/** This function creates a series of spec files with paste in macros used
* instead of normal keys. The tests will ensure that the correct keys are
* getting written to the keys table by running read_spec() and then using
* apop functions to verify that the keys are indeed in the spec file
*/
void pastein_tests(){
char *spec1;
Asprintf(&spec1, "1.spec");
char *spec2;
Asprintf(&spec2, "2.spec");
char *spec3;
Asprintf(&spec3, "3.spec");
char *spec4;
Asprintf(&spec4, "4.spec");
char *spec5;
Asprintf(&spec5, "5.spec");
/* Standard test here: creating a macro with a few sub keys and calling it on its own
* in the impute key. If something goes wrong here it's because there's something
* fundamentally wrong with the paste in macro (because there's only one so there's
* nothing too complex going on).
*/
write_a_file(spec1,
"\n"
"database: demo.db\n"
"verbose: 2\n"
"catagesex{\n"
" min group size: 3\n"
" draw count: 3\n"
" seed: 2332\n"
" categories {\n"
" CATAGE\n"
" SEX\n"
" }\n"
"}\n"
"\n"
"input {\n"
" input file: dc_pums_08.csv\n"
" output table: dc \n "
" overwrite: y \n "
"} \n "
" \n"
"fields { \n"
"SCHL: int 0-24 \n"
"WAGP: real\n"
"\n}"
"impute{\n"
" input table: viewdc\n"
" output table: imputes\n"
" paste in: catagesex\n"
" method: hot deck\n"
" output vars: SCHL\n"
"}\n"
"impute{\n"
" input table: viewdc\n"
" output table: imputes\n"
" paste in: catagesex\n"
" method: hot deck\n"
" output vars: WAGP\n"
"}\n"
);
/* Creating test here that uses two macros that are used concurrently but that do not
* call each other. tables{...} and catagesex{...} are each used in impute{...} but
* they do not "paste each other in". This will be tested in spec 3.
*/
write_a_file(spec2,
"\n"
"database: demo.db\n"
"verbose: 2\n"
"catagesex{\n"
" min group size: 3\n"
" draw count: 3\n"
" seed: 2332\n"
" categories {\n"
" CATAGE\n"
" SEX\n"
" }\n"
"}\n"
"tables{\n"
" input table: viewdc\n"
" output table: impuTable\n" //To account for analysts who like camel case
"}\n"
"\n"
"input {\n"
" paste in: tables\n"
" input file: dc_pums_08.csv\n"
" output table: dc \n "
" overwrite: y \n "
"} \n "
" \n"
"fields { \n"
"SCHL: int 0-24 \n"
"WAGP: real\n"
"\n}"
"impute{\n"
" paste in: tables\n"
" paste in: catagesex\n"
" method: hot deck\n"
" output vars: SCHL\n"
"}\n"
"impute{\n"
" paste in: tables\n"
" paste in: catagesex\n"
" method: hot deck\n"
" output vars: WAGP\n"
"}\n"
);
/* More complicated test that tests the ability of a macro to use another macro in its
* own definition. For instance, it tests something along the lines of
* catagesex{paste in: impute stuff \n paste in: categories}
*/
write_a_file(spec3,
"\n"
"database: demo.db\n"
"verbose: 2\n"
"imputestuff{\n"
" min group size: 3\n"
" draw count: 3\n"
" seed: 2332\n"
"}\n"
"categoriesstuff {\n"
" categories{\n"
" CATAGE\n"
" SEX\n"
" }\n"
"}\n"
"catagesex{\n"
" paste in: imputestuff\n"
" paste in: categoriesstuff\n"
"}\n"
"\n"
"input {\n"
" input file: dc_pums_08.csv\n"
" output table: dc \n "
" overwrite: y \n "
"} \n "
" \n"
"fields { \n"
"SCHL: int 0-24 \n"
"WAGP: real\n"
"\n}"
"impute{\n"
" input table: viewdc\n"
" output table: imputes\n"
" paste in: catagesex\n"
" method: hot deck\n"
" output vars: SCHL\n"
"}\n"
"impute{\n"
" input table: viewdc\n"
" output table: imputes\n"
" paste in: catagesex\n"
" method: hot deck\n"
" output vars: WAGP\n"
"}\n"
);
/* Tests whether it's possible to create a macro that comprises the entire spec file
* (which, of course, is then pasted in on its own). This includes other macros that
* are written within the overarching macro itself. Possibly overkill? But I think
* it's worth it to test given that different analysts might include big portions of
* the spec file separately and could decide to use a macro to do so.
*/
write_a_file(spec4,
"\n"
"database: demo.db\n"
"wholeSpec{\n"
"verbose: 2\n"
"catagesex{\n"
" min group size: 3\n"
" draw count: 3\n"
" seed: 2332\n"
" categories {\n"
" CATAGE\n"
" SEX\n"
" }\n"
"}\n"
"\n"
"input {\n"
" input file: dc_pums_08.csv\n"
" output table: dc \n "
" overwrite: y \n "
"} \n "
" \n"
"fields { \n"
"SCHL: int 0-24 \n"
"WAGP: real\n"
"\n}"
"impute{\n"
" input table: viewdc\n"
" output table: imputes\n"
" paste in: catagesex\n"
" method: hot deck\n"
" output vars: SCHL\n"
"}\n"
"impute{\n"
" input table: viewdc\n"
" output table: imputes\n"
" paste in: catagesex\n"
" method: hot deck\n"
" output vars: WAGP\n"
"}\n"
"}\n"
"paste in: wholeSpec\n"
);
char *db_dummy;
char *imp_min_grp, *imp_drw_cnt, *imp_seed, *imp_categories;
read_spec(&spec1, &db_dummy);
Asprintf(&imp_min_grp, "impute/min group size");
Asprintf(&imp_drw_cnt, "impute/draw count");
Asprintf(&imp_seed, "impute/seed");
Asprintf(&imp_categories, "impute/categories");
apop_data *spec1_keys1 = apop_query_to_text("select * from keys where key like "
"'impute/m%%'");
printf("spec1_keys1->text[0][0] is given by: %s.\n", spec1_keys1->text[0][0]);
assert(!strcmp(imp_min_grp, spec1_keys1->text[0][0]));
apop_data *spec1_keys2 = apop_query_to_text("select * from keys where key like "
"'impute/d%%'");
printf("spec1_keys2->text[0][0] is given by: %s.\n", spec1_keys2->text[0][0]);
assert(!strcmp(imp_drw_cnt, spec1_keys2->text[0][0]));
apop_data *spec1_keys3 = apop_query_to_text("select * from keys where key like "
"'impute/s%%'");
printf("spec1_keys3->text[0][0] is given by: %s.\n", spec1_keys3->text[0][0]);
assert(!strcmp(imp_seed, spec1_keys3->text[0][0]));
apop_data *spec1_keys4 = apop_query_to_text("select * from keys where key like "
"'impute/c%%'");
printf("spec1_keys4->text[0][0] is given by: %s.\n", spec1_keys4->text[0][0]);
assert(!strcmp(imp_categories, spec1_keys4->text[0][0]));
apop_data_free(spec1_keys1);
apop_data_free(spec1_keys2);
apop_data_free(spec1_keys3);
apop_data_free(spec1_keys4);
read_spec(&spec2, &db_dummy);
char *inpt_inpt_table;
char *inpt_otpt_table;
Asprintf(&inpt_inpt_table, "input/input table");
Asprintf(&inpt_otpt_table, "input/output table");
apop_data *spec2_keys1 = apop_query_to_text("select * from keys where key like "
"'impute/m%%'");
printf("spec2_keys1->text[0][0] is given by: %s.\n", spec2_keys1->text[0][0]);
assert(!strcmp(imp_min_grp, spec2_keys1->text[0][0]));
apop_data *spec2_keys2 = apop_query_to_text("select * from keys where key like "
"'impute/d%%'");
printf("spec2_keys2->text[0][0] is given by: %s.\n", spec2_keys2->text[0][0]);
assert(!strcmp(imp_drw_cnt, spec2_keys2->text[0][0]));
apop_data *spec2_keys3 = apop_query_to_text("select * from keys where key like "
"'impute/s%%'");
printf("spec2_keys3->text[0][0] is given by: %s.\n", spec2_keys3->text[0][0]);
assert(!strcmp(imp_seed, spec2_keys3->text[0][0]));
apop_data *spec2_keys4 = apop_query_to_text("select * from keys where key like "
"'impute/c%%'");
printf("spec2_keys4->text[0][0] is given by: %s.\n", spec2_keys4->text[0][0]);
assert(!strcmp(imp_categories, spec2_keys4->text[0][0]));
apop_data *spec2_keys5 = apop_query_to_text("select * from keys where key like "
"'input/input t%%'");
printf("spec2_keys5->text[0][0] is given by: %s.\n", spec2_keys5->text[0][0]);
assert(!strcmp(inpt_inpt_table, spec2_keys5->text[0][0]));
apop_data *spec2_keys6 = apop_query_to_text("select * from keys where key like "
"'input/output t%%'");
printf("spec2_keys6->text[0][0] is given by: %s.\n", spec2_keys6->text[0][0]);
assert(!strcmp(inpt_otpt_table, spec2_keys6->text[0][0]));
apop_data_free(spec2_keys1);
apop_data_free(spec2_keys2);
apop_data_free(spec2_keys3);
apop_data_free(spec2_keys4);
apop_data_free(spec2_keys5);
apop_data_free(spec2_keys6);
read_spec(&spec3, &db_dummy);
apop_data *spec3_keys1 = apop_query_to_text("select * from keys where key like "
"'impute/m%%'");
printf("spec3_keys1->text[0][0] is given by: %s.\n", spec3_keys1->text[0][0]);
assert(!strcmp(imp_min_grp, spec3_keys1->text[0][0]));
apop_data *spec3_keys2 = apop_query_to_text("select * from keys where key like "
"'impute/d%%'");
printf("spec3_keys2->text[0][0] is given by: %s.\n", spec3_keys2->text[0][0]);
assert(!strcmp(imp_drw_cnt, spec3_keys2->text[0][0]));
apop_data *spec3_keys3 = apop_query_to_text("select * from keys where key like "
"'impute/s%%'");
printf("spec3_keys3->text[0][0] is given by: %s.\n", spec3_keys3->text[0][0]);
assert(!strcmp(imp_seed, spec3_keys3->text[0][0]));
apop_data *spec3_keys4 = apop_query_to_text("select * from keys where key like "
"'impute/c%%'");
printf("spec3_keys4->text[0][0] is given by: %s.\n", spec3_keys4->text[0][0]);
assert(!strcmp(imp_categories, spec3_keys4->text[0][0]));
apop_data_free(spec3_keys1);
apop_data_free(spec3_keys2);
apop_data_free(spec3_keys3);
apop_data_free(spec3_keys4);
/* This is spec file that tests whether paste in works for pasting in the entire spec
* file (without the database -- pasting in database has not been tested yet). spec 4
* paste in stuff is tested by just testing for an assortment of keys.
*/
read_spec(&spec4, &db_dummy);
/* DV - ATTENTION:
* This test is failing right now so I've put in an if statement below to exit when
* there's no impute key to avoid a segfault in the testing. We need to fix the bug
* that is preventing paste in from allowing an entire spec file (minus the database)
* to be pasted in.
*/
apop_data *spec4_keys1 = apop_query_to_text("select * from keys where key like "
"'impute/m%%'");
if(get_key_word("impute", NULL) == NULL) return;
printf("spec4_keys1->text[0][0] is given by: %s.\n", spec4_keys1->text[0][0]);
assert(!strcmp(imp_min_grp, spec4_keys1->text[0][0]));
apop_data *spec4_keys2 = apop_query_to_text("select * from keys where key like "
"'impute/d%%'");
printf("spec4_keys2->text[0][0] is given by: %s.\n", spec4_keys2->text[0][0]);
assert(!strcmp(imp_drw_cnt, spec4_keys2->text[0][0]));
apop_data *spec4_keys3 = apop_query_to_text("select * from keys where key like "
"'impute/s%%'");
printf("spec4_keys3->text[0][0] is given by: %s.\n", spec4_keys3->text[0][0]);
assert(!strcmp(imp_seed, spec4_keys3->text[0][0]));
apop_data *spec4_keys4 = apop_query_to_text("select * from keys where key like "
"'impute/c%%'");
printf("spec4_keys4->text[0][0] is given by: %s.\n", spec4_keys4->text[0][0]);
assert(!strcmp(imp_categories, spec4_keys4->text[0][0]));
apop_data *spec4_keys5 = apop_query_to_text("select * from keys where key like "
"'input/input t%%'");
printf("spec4_keys5->text[0][0] is given by: %s.\n", spec4_keys5->text[0][0]);
assert(!strcmp(inpt_inpt_table, spec4_keys5->text[0][0]));
apop_data *spec4_keys6 = apop_query_to_text("select * from keys where key like "
"'input/output t%%'");
printf("spec4_keys6->text[0][0] is given by: %s.\n", spec4_keys6->text[0][0]);
assert(!strcmp(inpt_otpt_table, spec4_keys6->text[0][0]));
apop_data_free(spec4_keys1);
apop_data_free(spec4_keys2);
apop_data_free(spec4_keys3);
apop_data_free(spec4_keys4);
apop_data_free(spec4_keys5);
apop_data_free(spec4_keys6);
free(imp_min_grp);
free(imp_drw_cnt);
free(imp_seed);
free(imp_categories);
free(inpt_inpt_table);
free(inpt_otpt_table);
free(spec1);
free(spec2);
free(spec3);
free(spec4);
free(spec5);
printf("Reached end of test.\n");
}
