frameT *read_csv (char *file_name)
{
FILE *fp;
unsigned char buf[CSV_BUF_SIZE];
unsigned char header_buf[CSV_BUF_SIZE];
unsigned char *row[MAX_COLUMNS];
unsigned char *header[MAX_COLUMNS];
int ncol, cols;
int n, i;
frameT *frame;
csvT *csv;
csv = open_csv (file_name);
fp = csv->fp;
// Parse header
n = csv_row_fread (fp, header_buf, CSV_BUF_SIZE, header, MAX_COLUMNS, ',', CSV_TRIM | CSV_QUOTES, &ncol);
if (n <= 0)
{
fprintf (stderr, "Failed to read header from CSV file\n");
exit (-1);
}
// Parse the first data row to determine data types
n = csv_row_fread (fp, buf, CSV_BUF_SIZE, row, ncol, ',', CSV_TRIM | CSV_QUOTES, &cols);
if (n <= 0)
{
fprintf (stderr, "File has no data\n");
exit (-1);
}
frame = new_frame (file_name, ncol);
frame->csv = csv;
for (i=0; i<ncol; i++) init_column (frame, i, (char *)header[i], guess_type((char *) row[i]));
for (i=0; i<ncol; i++) column_init_data (frame, i, csv->est_rows);
frame->allocated_rows = csv->est_rows;
frame->region_rows = (unsigned char *) malloc (frame->allocated_rows);
if (!frame->region_rows)
{
fprintf (stderr, "Failed to allocate region_rows\n");
exit (-1);
}
frame->allocated_region_rows = frame->allocated_rows;
if (csv->est_rows < 10000)
{
load_all_rows (frame);
} else
{
load_random_rows (frame, 1.0);
}
return (frame);
}
int
symload(struct idl *idl, const char *filename)
{
FILE *in;
int n;
unsigned char buf[512], *row[4];
if ((in = fopen(filename, "r")) == NULL) {
PMNF(errno, ": %s", filename);
return -1;
}
while ((n = csv_row_fread(in, buf, 512, row, 4, '\t', CSV_TRIM | CSV_QUOTES)) > 0) {
struct sym *sym;
if (*row[0] == '\0' || *row[0] == '#') {
continue;
}
if ((sym = symadd(idl, row[0], row[1], row[2], atoi(row[3]))) == NULL) {
AMSG("");
return -1;
}
sym->flags |= FLAGS_PRIMATIVE;
}
fclose(in);
if (n == -1) {
AMSG("");
return -1;
}
return 0;
}
void load_row (frameT *frame)
{
csvT *csv = frame->csv;
int bytes;
int i, cols;
unsigned char buf[CSV_BUF_SIZE];
unsigned char *row[MAX_COLUMNS];
bytes = csv_row_fread (csv->fp, buf, CSV_BUF_SIZE, row, frame->columns, ',', CSV_TRIM | CSV_QUOTES, &cols);
if (bytes < 0)
{
fprintf (stderr, "Failed to read csv line!\n");
exit (-1);
}
if (cols != frame->columns)
{
fprintf (stderr, "Expected %d columns, got %d\n", frame->columns, cols);
for (i=0; i<cols; i++) printf ("%s ", row[i]);
printf ("\n");
exit (-1);
}
// TODO: UPDATE ESTIMATES
for (i=0; i<cols; i++) column_add_data (frame, i, (char *)row[i]);
frame->rows++;
if (frame->rows >= frame->allocated_rows * 0.8)
{
unsigned long new_alloc = frame->rows * 1.5;
for (i=0; i<cols; i++) column_realloc_data (frame, i, new_alloc);
free (frame->region_rows);
frame->region_rows = calloc (new_alloc, sizeof(unsigned char));
if (!frame->region_rows)
{
fprintf (stderr, "Failed to realloc region_rows\n");
exit (-1);
}
frame->allocated_region_rows = new_alloc;
frame->allocated_rows = new_alloc;
}
// We don't over-allocate transform data, so we must force reallocs whenever
// we load new data.
for (i=0; i<cols; i++) column_wipe_transforms (frame, i);
if (frame->nn_distance)
{
free (frame->nn_distance);
frame->nn_distance = NULL;
}
}
int
CfgQueryStringExamples(int verbose, struct cfg *cfg, char *args[])
{
unsigned char buf[1024], *row[10];
FILE *in;
int ret;
if ((in = fopen(args[0], "r")) == NULL) {
PMNO(errno);
return -1;
}
while ((ret = csv_row_fread(in, buf, 1024, row, 10, ',', CSV_QUOTES | CSV_TRIM)) > 0) {
int success = atoi(row[0]);
tcase_printf(verbose, "%s:\n", row[1]);
cfg_clear(cfg);
if (cfg_load_cgi_query_string(cfg, row[1], row[1] + tcslen(row[1])) == -1) {
if (success) {
ret = -1;
AMSG("");
goto out;
}
} else if (!success) {
ret = -1;
AMSG("Supposed to fail");
goto out;
}
if (verbose) {
iter_t iter;
const tchar *name;
tchar dst[512];
cfg_iterate(cfg, &iter);
while ((name = cfg_next(cfg, &iter))) {
if (cfg_get_str(cfg, dst, 512, NULL, name) == -1) {
errno = ENOENT;
PMNO(errno);
return -1;
}
tcase_printf(verbose, "\t%s=%s\n", name, dst);
}
}
}
out:
fclose(in);
return ret;
}
int
run_suite(const char *cfgname, const char *csvname, int testnums[], int tlen)
{
struct cfg *cfg;
FILE *csv;
unsigned char buf[4096], *row[32];
int tnum, ti, ret, flags;
if ((cfg = cfg_new(NULL)) == NULL ||
cfg_load(cfg, cfgname)) {
AMSG(": %s", cfgname);
return -1;
}
if ((csv = fopen(csvname, "r")) == NULL) {
PMNF(errno, ": %s", csvname);
return -1;
}
while ((ret = csv_row_fread(csv, buf, 4096, row, 32, ',', CSV_TRIM | CSV_QUOTES)) > 0) {
if ((flags = strtol(row[1], NULL, 10)) == LONG_MIN || flags == LONG_MAX) {
PMNF(errno, ": Invalid test flags: %s", row[1]);
return -1;
}
if (flags == 0) {
continue;
}
if ((tnum = strtol(row[0], NULL, 10)) == LONG_MIN || tnum == LONG_MAX) {
PMNF(errno, ": Invalid test number: %s", row[0]);
return -1;
}
if (tlen) {
for (ti = 0; ti < tlen; ti++) {
if (tnum == testnums[ti]) {
break;
}
}
if (ti == tlen) {
continue;
}
}
for (ti = 0; test_tbl[ti].test; ti++) {
if (tnum == test_tbl[ti].num) {
break;
}
}
if (test_tbl[ti].test == NULL) {
PMSG("Test not found: %d", tnum);
return -1;
}
errno = 0;
fprintf(stderr, "%d:%s: ", tnum, row[2]);
fflush(stderr);
if (test_tbl[ti].test(flags > 1 ? flags : 0, cfg, (char **)row + 4) == 0) {
fprintf(stderr, "pass\n");
} else {
MMSG("%d failed", tnum);
}
}
fclose(csv);
cfg_del(cfg);
return ret;
}
static int build_table_from_csv_fh(FILE *in, GtkWidget **ret_view, GtkTreeModel **ret_store, GtkProgressBar *progress)
{
GType *column_types;
int n, ncol, nrow=0;
int sep = ',';
GtkListStore *store=NULL;
GtkWidget *view=NULL;
if (csv_row_fread(in,
(unsigned char *)csv_buf,
CSV_BUFSIZE,
(unsigned char **)csv_row,
CSV_COLS,
sep,
CSV_QUOTES) > 0) {
/* Create model with number of columns matching the csv input */
for(ncol=0; csv_row[ncol]; ncol++);
column_types = malloc(sizeof(GType) * ncol);
for (n=0; n<ncol; n++) {
column_types[n] = G_TYPE_STRING;
}
store = gtk_list_store_newv(ncol, column_types);
free(column_types);
/* Create view, add colums */
GtkCellRenderer *renderer;
GtkTreeViewColumn *column;
view = gtk_tree_view_new_with_model (GTK_TREE_MODEL(store));
for (n = 0; n < ncol; n++) {
renderer = gtk_cell_renderer_text_new();
column = gtk_tree_view_column_new_with_attributes(csv_row[n], renderer, "text", n, NULL);
/* gtk_tree_view_column_set_sort_column_id(column, n); */
/* gtk_tree_view_column_set_reorderable(column, TRUE); */
gtk_tree_view_column_set_resizable(column, TRUE);
gtk_tree_view_insert_column(GTK_TREE_VIEW(view), column, n);
}
/* Populate model */
GtkTreeIter iter;
if (progress) {
gtk_progress_bar_set_fraction(progress, 0.0);
}
while (csv_row_fread(in,
(unsigned char *)csv_buf,
CSV_BUFSIZE,
(unsigned char **)csv_row,
CSV_COLS,
sep,
CSV_QUOTES) > 0) {
gtk_list_store_append(store, &iter);
for(n = 0; n < ncol; n++) {
gtk_list_store_set(store, &iter, n, csv_row[n], -1);
}
if (!(nrow % 256)) {
while (gtk_events_pending()) gtk_main_iteration();
}
gtk_main_iteration_do(FALSE);
nrow++;
if (progress) {};
}
}
g_object_unref(store); /* So only the treeview ref remains - it should be deleted when treeview is destroyed */
if (ret_store) *ret_store = GTK_TREE_MODEL(store);
if (ret_view) *ret_view = view;
return nrow;
}
