int ptm_lib_init_msg(ptm_lib_handle_t *hdl, int cmd_id, int type, void *in_ctxt,
void **out_ctxt)
{
ptm_lib_msg_ctxt_t *p_ctxt;
ptm_lib_msg_ctxt_t *p_in_ctxt = in_ctxt;
csv_t *csv;
csv_record_t *rec, *d_rec;
/* Initialize csv for using discrete record buffers */
csv = csv_init(NULL, NULL, PTMLIB_MSG_SZ);
if (!csv) {
ERRLOG("%s: Could not allocate csv \n", __FUNCTION__);
return -1;
}
rec = _ptm_lib_encode_header(csv, NULL, 0, PTMLIB_MSG_VERSION, type,
cmd_id, hdl->client_name);
if (!rec) {
ERRLOG("%s: Could not allocate record \n", __FUNCTION__);
csv_clean(csv);
csv_free(csv);
return -1;
}
p_ctxt = calloc(1, sizeof(*p_ctxt));
if (!p_ctxt) {
ERRLOG("%s: Could not allocate context \n", __FUNCTION__);
csv_clean(csv);
csv_free(csv);
return -1;
}
p_ctxt->csv = csv;
p_ctxt->cmd_id = cmd_id;
p_ctxt->type = type;
*(ptm_lib_msg_ctxt_t **)out_ctxt = p_ctxt;
/* caller supplied a context to initialize with? */
if (p_in_ctxt) {
/* insert the hdr rec */
rec = csv_record_iter(p_in_ctxt->csv);
csv_clone_record(p_in_ctxt->csv, rec, &d_rec);
csv_insert_record(csv, d_rec);
/* insert the data rec */
rec = csv_record_iter_next(rec);
csv_clone_record(p_in_ctxt->csv, rec, &d_rec);
csv_insert_record(csv, d_rec);
}
return 0;
}
int loadSameNameUnoverlapped()
{
FILE *fp;
int len=0;
struct csv_parser *p;
char buf[1024];
size_t bytes_read;
const char *file_name="data/NewSameNameUnoverlapped.csv";
//const char *file_name="data/SameNameUnoverlapped.csv";
if((p=(struct csv_parser *)malloc(sizeof(struct csv_parser))) == 0) return -1;
if ((fp = fopen(file_name,"r"))== NULL)
fprintf(stderr, "Failed to open %s\n",file_name);
csv_init(p, (unsigned char)0);
while ((bytes_read=fread(buf, 1, 1024, fp)) > 0)
if (csv_parse(p, buf, bytes_read, col, row,&len) != bytes_read)
{
fprintf(stderr, "Error while parsing file: %s\n", csv_strerror(csv_error(p)));
}
qsort(aids,len,sizeof(int),intcmp);
aidlen = len;
csv_free(p);
fclose(fp);
for(int i=0;i<100;i++) printf("%d | ",aids[i]);
printf("\n%d",aidlen);
return 1;
}
int SOMAexternal (SOMAsetup *ssetup, FitnessFunction ffunc, int nparams, Parameter *params, const char *file, SOMAreturn* sr) {
Array2D array;
char *buf;
long len;
struct csv_parser csvp;
clock_t begin, end;
array_init (&array);
array_append_column(&array); //blank column for external fitness function
array.currentcolumn++;
//Parse CSV file
if (csv_init (&csvp, CSV_APPEND_NULL)!=0)
return ERR;
if ((buf=read_file_into_buffer (file, &len))==NULL)
return ERR_FILE;
csv_parse (&csvp, buf, len, csvcallback1, csvcallback2e, (void*) &array);
csv_fini (&csvp, csvcallback1, csvcallback2e, (void*) &array);
csv_free (&csvp);
//array_print (&array); //debug
//soma_array=&array;
//Initialize output struct
SOMAreturn_init (sr, ssetup->Migrations, nparams);
begin=clock();
//SOMA call
SOMA2 (ssetup, nparams, params, sr, &array, ffunc);
end=clock();
sr->time=(double) (end-begin)/CLOCKS_PER_SEC;
array_delete (&array);
return OK;
}
int ptm_lib_complete_msg(ptm_lib_handle_t *hdl, void *ctxt, char *buf, int *len)
{
ptm_lib_msg_ctxt_t *p_ctxt = ctxt;
csv_t *csv;
csv_record_t *rec;
if (!p_ctxt) {
ERRLOG("%s: no context \n", __FUNCTION__);
return -1;
}
csv = p_ctxt->csv;
rec = csv_record_iter(csv);
_ptm_lib_encode_header(csv, rec, (csvlen(csv) - PTMLIB_MSG_HDR_LEN),
PTMLIB_MSG_VERSION, p_ctxt->type, p_ctxt->cmd_id,
hdl->client_name);
/* parse csv contents into string */
if (buf && len) {
if (csv_serialize(csv, buf, *len)) {
ERRLOG("%s: cannot serialize\n", __FUNCTION__);
return -1;
}
*len = csvlen(csv);
}
csv_clean(csv);
csv_free(csv);
free(p_ctxt);
return 0;
}
int SOMAascii (SOMAsetup *ssetup, const char *expr, int nparams, Parameter *params, const char *file, SOMAreturn* sr) {
Array2D array;
int i;
char *buf;
long len;
struct csv_parser csvp;
clock_t begin, end;
MuExpr me;
array_init(&array);
//Parse CSV file
if (csv_init (&csvp, CSV_APPEND_NULL)!=0)
return ERR;
if ((buf=read_file_into_buffer (file, &len))==NULL)
return ERR_FILE;
csv_parse (&csvp, buf, len, csvcallback1, csvcallback2, (void*) &array);
csv_fini (&csvp, csvcallback1, csvcallback2, (void*) &array);
csv_free (&csvp);
//array_print (&array); //debug
muexpr_init (&me, expr);
for (i=0; i<nparams; i++)
if (defpar (&me, params+i, i)!=OK) {
return ERR_PARAMS;
array_delete (&array);
muexpr_free (&me);
}
findvars (&me, VAR_NAME_PREFIX);
//CVS file and expression don't match
if (me.nvars!=array.ncolumns-1) {
array_delete (&array);
muexpr_free (&me);
return ERR_COLUMNS;
}
muexpr_eval (&me);
if (mupError (me.hparser)) {
array_delete (&array);
muexpr_free (&me);
return ERR_EXPR;
}
//Initialize output struct
SOMAreturn_init (sr, ssetup->Migrations, nparams);
begin=clock();
//SOMA call
SOMA (ssetup, nparams, params, sr, &array, &me);
end=clock();
sr->time=(double) (end-begin)/CLOCKS_PER_SEC;
array_delete (&array);
muexpr_free (&me);
return OK;
}
static void
release_ecsv_parser(UNUSED(ErlNifEnv * env), void *obj)
{
//enif_fprintf(stderr, "Releasing parser resource\n");
ecsv_parser_t *parser = (ecsv_parser_t *)obj;
enif_free_env(parser->env);
if (parser->current_line.fields) enif_free(parser->current_line.fields);
csv_free(&parser->p);
//enif_fprintf(stderr, "Released parser resource\n");
}
int main (int argc, char *argv[]) {
char buf[1024];
size_t i;
struct csv_parser p;
FILE *infile, *outfile;
csv_init(&p, 0);
if (argc != 3) {
fprintf(stderr, "Usage: csv_fix infile outfile\n");
return EXIT_FAILURE;
}
if (!strcmp(argv[1], argv[2])) {
fprintf(stderr, "Input file and output file must not be the same!\n");
exit(EXIT_FAILURE);
}
infile = fopen(argv[1], "rb");
if (infile == NULL) {
fprintf(stderr, "Failed to open file %s: %s\n", argv[1], strerror(errno));
exit(EXIT_FAILURE);
}
outfile = fopen(argv[2], "wb");
if (outfile == NULL) {
fprintf(stderr, "Failed to open file %s: %s\n", argv[2], strerror(errno));
fclose(infile);
exit(EXIT_FAILURE);
}
while ((i=fread(buf, 1, 1024, infile)) > 0) {
if (csv_parse(&p, buf, i, cb1, cb2, outfile) != i) {
fprintf(stderr, "Error parsing file: %s\n", csv_strerror(csv_error(&p)));
fclose(infile);
fclose(outfile);
remove(argv[2]);
exit(EXIT_FAILURE);
}
}
csv_fini(&p, cb1, cb2, outfile);
csv_free(&p);
if (ferror(infile)) {
fprintf(stderr, "Error reading from input file");
fclose(infile);
fclose(outfile);
remove(argv[2]);
exit(EXIT_FAILURE);
}
fclose(infile);
fclose(outfile);
return EXIT_SUCCESS;
}
void genericLineBasedParsing(
std::istream &file,
field_cb_t cb_per_field,
line_cb_t cb_per_line,
void *data,
const csv::params ¶ms) {
struct csv_parser parser;
if (!csv_init(&parser, 0)) {
csv_set_opts(&parser, CSV_APPEND_NULL);
csv_set_delim(&parser, params.getDelimiter());
std::string line;
int line_start = params.getLineStart();
if (line_start != 1) {
while (line_start > 1) {
std::getline(file, line);
--line_start;
}
}
int lineCount = 0;
while (std::getline(file, line)) {
++lineCount;
line.append("\n");
if (csv_parse(&parser,
line.c_str(),
line.size(),
cb_per_field,
cb_per_line,
data) != line.size()) {
throw ParserError(csv_strerror(csv_error(&parser)));
}
if (params.getLineCount() != -1 && lineCount >= params.getLineCount())
break;
if (file.bad())
break;
}
csv_fini(&parser,
cb_per_field,
cb_per_line,
data);
}
csv_free(&parser);
}
int main (int argc, char* argv[]) {
struct csv_parser parser = {0};
csv_init (&parser, CSV_APPEND_NULL);
char *buf = (char*)malloc (READ_SZ);
size_t buflen = READ_SZ;
int count = 0;
while ((buflen = read (0, buf, READ_SZ)) > 0) {
csv_parse (&parser, buf, buflen, field_count, 0, &count);
}
printf ("%d\n", count);
free (buf);
csv_free (&parser);
return EXIT_SUCCESS;
}
CLUSTER_ITEM* KMeansLoadCSV(char *filename, int labelColumn, int startColumn, int featureCount) {
FILE *fp;
char buf[1024];
size_t bytes_read;
struct _KMeansStructCSV c;
struct csv_parser p;
CLUSTER_ITEM *result;
/* Setup csvlib to read the CSV file */
if (csv_init(&p, CSV_APPEND_NULL) != 0) exit(EXIT_FAILURE);
fp = fopen(filename, "rb");
if (!fp)
{
printf("Could not open: %s\n", filename);
exit(EXIT_FAILURE);
}
c.row = 0;
c.col = 0;
c.startCol = startColumn;
c.featureCount = featureCount;
c.labelCol = labelColumn;
c.item = c.prevItem = c.firstItem = NULL;
c.features = (double*)calloc(featureCount,sizeof(double));
/* Loop over the contents. It is important to note that we are not reading line by
line, at this level. Rather, we are passing blocks off to csvlib. Then csvlib
calls our two callbacks as needed. */
while ((bytes_read=fread(buf, 1, 1024, fp)) > 0) {
if (csv_parse(&p, buf, bytes_read, _KMeansCallbackColumn, _KMeansCallbackRow, &c) != bytes_read) {
fprintf(stderr, "Error while parsing file: %s\n",
csv_strerror(csv_error(&p)) );
exit(EXIT_FAILURE);
}
}
result = c.firstItem;
/* Handle any final data. May call the callbacks once more */
csv_fini(&p, _KMeansCallbackColumn, _KMeansCallbackRow, &c);
/* Cleanup */
free(c.features);
fclose(fp);
csv_free(&p);
return result;
}
void
database_read(database_t *database) {
size_t i, total;
csv_t *csv;
csv_row_t *csv_row;
csv_fields_t *csv_fields;
record_t *record;
csv = csv_new(database->filename);
csv_row = csv_row_new(NULL, NULL);
total = 0;
while(csv_get_row(csv, csv_row)) {
csv_fields = csv_fields_new(database->num_fields);
csv_get_fields(csv_fields, csv_row, database->sep);
record = record_new(database->num_fields);
for(i = 0; i < database->num_fields; i++) {
record_add_field(record, csv_fields->fields[i]);
}
csv_fields_deep_free(csv_fields);
array_push(database->records, record);
total++;
if(!(total % 1000000)) {
printf(
"Registros lidos: %d de %d (%2.2f%%)\n",
(int) total,
(int) database->num_rows,
100.0 * total / database->num_rows
);
}
}
printf(
"Registros lidos: %d de %d (%2.2f%%)\n",
(int) total,
(int) database->num_rows,
100.0 * total / database->num_rows
);
csv_row_free(csv_row);
csv_free(csv);
}
/* ==================================================================
* Parser for comma-separated argument list
* ================================================================== */
void ParseVarList(int nLn, FILE *flp, char *dataName, char *leftPart, char *argString)
{
struct csv_parser p;
unsigned char options = 0;
LIST_DATA ldata;
// fill in data for the callback
memset(&ldata, '\x0', sizeof(LIST_DATA));
ldata.nLn = nLn;
ldata.flp = flp;
ldata.cnt = 0;
strcpy(ldata.dataName, dataName);
strcpy(ldata.lp, leftPart);
// Initialize csv parser
if (csv_init(&p, options) != 0)
{
fprintf(stderr, "Failed to initialize csv parser\n");
return;
}
// set white space, eol and delimiter
csv_set_space_func(&p, is_space_list);
csv_set_term_func(&p, is_term_list);
csv_set_delim(&p, ',');
unsigned int agrLen = strlen(argString);
fprintf(stderr, "ParseVarList: argString = %s argLen - %d\n", argString, agrLen);
memset(inputsLst, '\x0', sizeof(inputsLst));
InpCnt = 0;
if (csv_parse(&p, argString, strlen(argString), cbProcessListElement, NULL, &ldata) != agrLen)
{
fprintf(stderr, "ParseVarList: %s\n", csv_strerror(csv_error(&p)));
return;
}
csv_fini(&p, cbProcessListElement, NULL, &ldata);
csv_free(&p);
return;
}
/*
This example shows how to simply read a CSV file. This example reads the iris data set.
The output is shown here.
Reading CSV file: ./datasets/iris.csv
Field: "sepal_length"
Field: "sepal_width"
Field: "petal_length"
Field: "petal_width"
Field: "class"
Row done
Field: "5.1"
Field: "3.5"
Field: "1.4"
Field: "0.2"
Field: "Iris-setosa"
Row done
Field: "4.9"
Field: "3.0"
Field: "1.4"
Field: "0.2"
Field: "Iris-setosa"
Row done
Field: "4.7"
Field: "3.2"
Field: "1.3"
Field: "0.2"
Field: "Iris-setosa"
...
Row done
Field: "6.5"
Field: "3.0"
Field: "5.2"
Field: "2.0"
Field: "Iris-virginica"
Row done
Field: "6.2"
Field: "3.4"
Field: "5.4"
Field: "2.3"
Field: "Iris-virginica"
Row done
Field: "5.9"
Field: "3.0"
Field: "5.1"
Field: "1.8"
Field: "Iris-virginica"
Row done
755 fields, 151 rows
*/
void ExampleReadCSV(int argIndex, int argc, char **argv) {
char filename[FILENAME_MAX];
FILE *fp;
struct csv_parser p;
char buf[1024];
size_t bytes_read;
struct counts c = {0, 0};
if( argIndex>=argc ) {
LocateFile("iris.csv",filename,FILENAME_MAX);
} else {
strncpy(filename,argv[argIndex],FILENAME_MAX);
}
printf("Reading CSV file: %s\n", filename);
/* Setup csvlib to read the CSV file */
if (csv_init(&p, CSV_APPEND_NULL) != 0) exit(EXIT_FAILURE);
fp = fopen(filename, "rb");
if (!fp)
{
printf("Could not open: %s\n", filename);
exit(EXIT_FAILURE);
}
/* Loop over the contents. It is important to note that we are not reading line by
line, at this level. Rather, we are passing blocks off to csvlib. Then csvlib
calls our two callbacks as needed. */
while ((bytes_read=fread(buf, 1, 1024, fp)) > 0)
if (csv_parse(&p, buf, bytes_read, CallbackColumn, CallbackRow, &c) != bytes_read) {
fprintf(stderr, "Error while parsing file: %s\n",
csv_strerror(csv_error(&p)) );
exit(EXIT_FAILURE);
}
/* Handle any final data. May call the callbacks once more */
csv_fini(&p, CallbackColumn, CallbackRow, &c);
/* Print final stats on CSV file */
printf("%lu fields, %lu rows\n", c.fields, c.rows);
/* Cleanup */
fclose(fp);
csv_free(&p);
}
/**
* Free dynamically allocated stuff in our context
*/
void context_free(struct csv_context *ctx) {
// Free our pass through buffer
cbuf_free(ctx->csv_buf);
// Free group column buffer
if(ctx->gcol_buf) {
cbuf_free(ctx->gcol_buf);
}
// Free memory stored in our IO queue
fq_free(&ctx->io_queue);
// Free our CSV parser
csv_free(&ctx->parser);
// Free our thread storage
free(ctx->io_threads);
}
int ptm_lib_cleanup_msg(ptm_lib_handle_t *hdl, void *ctxt)
{
ptm_lib_msg_ctxt_t *p_ctxt = ctxt;
csv_t *csv;
if (!p_ctxt) {
ERRLOG("%s: no context \n", __FUNCTION__);
return -1;
}
csv = p_ctxt->csv;
csv_clean(csv);
csv_free(csv);
free(p_ctxt);
return 0;
}
void
bgpstream_csvfile_datasource_destroy(bgpstream_csvfile_datasource_t* csvfile_ds) {
bgpstream_debug("\t\tBSDS_CSVFILE: destroy csvfile_ds start");
if(csvfile_ds == NULL) {
return; // nothing to destroy
}
csvfile_ds->filter_mgr = NULL;
if(csvfile_ds->csvfile_file !=NULL)
{
free(csvfile_ds->csvfile_file);
}
if(&(csvfile_ds->parser) != NULL)
{
csv_free(&(csvfile_ds->parser));
}
free(csvfile_ds);
bgpstream_debug("\t\tBSDS_CSVFILE: destroy csvfile_ds end");
}
int main (void) {
struct csv_parser p;
int i;
char c;
csv_init(&p, 0);
while ((i=getc(stdin)) != EOF) {
c = i;
if (csv_parse(&p, &c, 1, cb1, cb2, NULL) != 1) {
fprintf(stderr, "Error: %s\n", csv_strerror(csv_error(&p)));
exit(EXIT_FAILURE);
}
}
csv_fini(&p, cb1, cb2, NULL);
csv_free(&p);
return EXIT_SUCCESS;
}
int main(int argc, char* argv[])
{
if (argc < 2)
{
fprintf(stderr, "Usage: %s <training-csv-file>\n", argv[0]);
return 1;
}
plotter = gnuplot_init();
csv_file* csv = csv_parse(argv[1]);
linear_regression* lr = linreg_run(csv);
printf("h(x) = %f + %fx\n", lr->theta0, lr->theta1);
plot_data_and_model(argv[1], lr);
linreg_free(lr);
csv_free(csv);
gnuplot_close(plotter);
return 0;
}
/* All the possible free()'s should be listed here.
This function should be invoked before returning the result to Ruby or raising an exception. */
void free_memory(struct csv_parser * cp, struct rcsv_metadata * meta) {
if (meta->only_rows != NULL) {
free(meta->only_rows);
}
if (meta->except_rows != NULL) {
free(meta->except_rows);
}
if (meta->row_defaults != NULL) {
free(meta->row_defaults);
}
if (meta->column_names != NULL) {
free(meta->column_names);
}
if (cp != NULL) {
csv_free(cp);
}
}
static int config_load(control_t *control)
{
struct csv_parser p;
FILE *fp;
char buf[MAX_BUF];
size_t bytes_read;
assert(control);
assert(control->configfile);
assert(control->entries == NULL);
assert(control->start == 0 && control->end == 0);
// open the file.
fp = fopen(control->configfile, "rb");
if (fp) {
if (csv_init(&p, 0) == 0) {
while ((bytes_read=fread(buf, 1, MAX_BUF, fp)) > 0) {
if (csv_parse(&p, buf, bytes_read, csv_data, csv_line, control) != bytes_read) {
fprintf(stderr, "Error while parsing file: %s\n",
csv_strerror(csv_error(&p)) );
exit(EXIT_FAILURE);
}
}
csv_fini(&p, csv_data, csv_line, control);
csv_free(&p);
}
fclose(fp);
return 0;
}
else {
return -1;
}
}
int main (int argc, char** argv)
{
paillier_pubkey_t* pkey;
paillier_prvkey_t* skey;
paillier_keygen(128,&pkey,&skey,&paillier_get_rand_devrandom);
void *context = zmq_ctx_new ();
struct opts options;
parse_options(argc,argv, &options);
if(options.size <= 0 || options.scale <= 0 || !options.fileset){
fprintf(stderr,"Size and scale must be greater than 0 and file must be set\n");
exit(EXIT_FAILURE);
}
struct classify_data data;
data.pub = pkey;
data.prv = skey;
data.maxcol = options.size;
data.scale_factor = options.scale;
data.texts = (paillier_plaintext_t**)malloc(options.size*sizeof(paillier_plaintext_t*));
data.col = 0;
data.correct = 0;
data.total = 0;
init_rand(data.rand,&paillier_get_rand_devurandom,pkey->bits / 8 + 1);
// Socket to talk to server
gmp_printf("n: %Zd, lambda: %Zd\n",pkey->n,skey->lambda);
void *requester = zmq_socket (context, ZMQ_REQ);
zmq_connect (requester, "ipc:///tmp/karma");
char* pubkeyhex = paillier_pubkey_to_hex(pkey);
s_send(requester,pubkeyhex);
char* recv = s_recv(requester);
free(recv);
free(pubkeyhex);
data.socket = requester;
char* file = options.file;
FILE* fp;
struct csv_parser p;
char buf[1024];
size_t bytes_read;
if(csv_init(&p,0)) {
fprintf(stderr, "Failed to initialize parser\n");
exit(EXIT_FAILURE);
}
fp = fopen(file,"rb");
if(!fp){
fprintf(stderr,"Failed to open classify file %s\n",strerror(errno));
exit(EXIT_FAILURE);
}
while ((bytes_read=fread(buf,1,1024,fp)) > 0){
if(!csv_parse(&p,buf,bytes_read,field_parsed,row_parsed,&data)){
fprintf(stderr, "Failed to parse file: %s\n",csv_strerror(csv_error(&p)));
}
}
csv_fini(&p,field_parsed,row_parsed,&data);
//fini took care of freeing the plaintexts
csv_free(&p);
free(data.texts);
gmp_randclear(data.rand);
printf("Correct(%i)/Total(%i) = %f\n",data.correct,data.total,data.correct/(data.total+0.0));
sleep (2);
zmq_close (requester);
zmq_ctx_destroy (context);
return 0;
}
cxCSV::~cxCSV()
{
rows->Release();
row->Release();
csv_free(&parser);
}
/* loads a CSV file from disk */
csv_file *csv_load(const char *filename, int *err, int *line)
{
csv_file *csv;
char *buffer, *p;
int r = 0, c = 0;
if(err) *err = ER_OK;
if(line) *line = 1;
if(!filename)
RETERR(ER_INV_PARAM);
csv = csv_create(0,0);
if(!csv)
RETERR(ER_MEM_FAIL);
/* My approach is to read the entire file into memory and then
* parse it from there.
* I Acknowledge that this is not the most efficient way of doing it,
* but it does simplify the parsing somewhat.
*/
buffer = my_readfile(filename);
if(!buffer)
ERREND(ER_IOR_FAIL);
for(p = buffer; *p;)
{
if(strchr(" \t",p[0]))
{
/* This is to prevent space between the start of a field
and a " from confusing the parser */
char *q;
for(q = p + 1; strchr(" \t",q[0]); q++);
if(q[0] == '\"')
p = q;
}
if(!strncmp(p,"\r\n", 2))
{
/* official line endings */
r++;
c = 0;
p+=2;
if(line) (*line)++;
}
else if(strchr("\r\n",p[0]))
{
/* alternative line endings */
r++;
c = 0;
p++;
if(line) (*line)++;
}
else if(*p == ',')
{
/* A new or empty field */
c++;
p++;
}
else if(*p == '\"')
{
/* A quoted field */
char *q = p, *s, *t;
do {
q++;
if(*q == '\0')
ERREND(ER_EXPECTED_EOS);
else if(q[0] == '\"' && q[1] == '\"')
q+=2;
} while(*q != '\"');
s = malloc(q - p);
if(!s)
ERREND(ER_MEM_FAIL);
t = s;
p++;
while(p < q)
{
if(*p == '\"' && *(p+1) == '\"')
{
*(t++) = *(p++);
p++;
}
else
*(t++) = *(p++);
}
*t = '\0';
/* Skip any whitespace after the closing quote */
for(p++; p[0] && !strchr(",\r\n",p[0]);p++)
if(!strchr(" \t",p[0]))
ERREND(ER_BAD_QUOTEEND);
csv_set_int(csv, r, c, s);
}
else
{
/* A normal field */
char *q, *s, *t;
#if TRIM_SPACES
/* Trim leading whitespace */
while(p[0] && strchr(" \t",p[0])) p++;
#endif
for(q = p;q[0] && !strchr(",\r\n",q[0]); q++);
s = malloc(q - p + 1);
if(!s)
ERREND(ER_MEM_FAIL);
t = s;
while(p < q)
{
*(t++) = *(p++);
}
*t = '\0';
#if TRIM_SPACES
/* Trim trailing whitespace */
while(t > s && strchr(" \t",t[-1]))
*(--t) = '\0';
#endif
csv_set_int(csv, r, c, s);
}
}
free(buffer);
return csv;
error:
csv_free(csv);
return NULL;
}
void genericParse(
/*std::istream &file,*/
std::string filename,
field_cb_t cb_per_field,
line_cb_t cb_per_line,
void *data,
const csv::params ¶ms
) {
// Open the file
typedef std::unique_ptr<std::FILE, int (*)(std::FILE *)> unique_file_ptr;
unique_file_ptr file(fopen(filename.c_str(), "rb"), fclose);
if (!file) {
throw ParserError(std::string("File Opening Failed") + std::strerror(errno));
}
struct csv_parser parser;
if (!csv_init(&parser, 0)) {
csv_set_opts(&parser, CSV_APPEND_NULL);
csv_set_delim(&parser, params.getDelimiter());
int line_start = params.getLineStart();
if (line_start > 1) {
int c;
do {
c = fgetc(file.get());
if ( c== '\n') --line_start;
} while (c!= EOF && line_start > 1);
}
// 1GB Buffer
size_t block_size;
if (getenv("HYRISE_LOAD_BLOCK_SIZE"))
block_size = strtoul(getenv("HYRISE_LOAD_BLOCK_SIZE"), nullptr, 0);
else
block_size = 1024 * 1024;
// Read from the buffer
size_t readBytes = 0;
char rdbuf[block_size];
// Read the file until we cannot extract more bytes
do {
readBytes = fread(rdbuf, 1, block_size, file.get());
if (csv_parse(&parser,
rdbuf,
readBytes,
cb_per_field,
cb_per_line,
data) != (size_t) readBytes) {
throw ParserError(csv_strerror(csv_error(&parser)));
}
} while (readBytes == block_size);
if (ferror(file.get())) {
throw ParserError("Could not read file");
}
csv_fini(&parser,
cb_per_field,
cb_per_line,
data);
}
csv_free(&parser);
}
int ptm_lib_process_msg(ptm_lib_handle_t *hdl, int fd, char *inbuf, int inlen,
void *arg)
{
int rc, len;
char client_name[32];
int cmd_id = 0, type = 0, ver = 0, msglen = 0;
csv_t *csv;
ptm_lib_msg_ctxt_t *p_ctxt = NULL;
len = _ptm_lib_read_ptm_socket(fd, inbuf, PTMLIB_MSG_HDR_LEN);
if (len <= 0)
return (len);
csv = csv_init(NULL, inbuf, PTMLIB_MSG_HDR_LEN);
if (!csv) {
DLOG("Cannot allocate csv for hdr\n");
return (-1);
}
rc = _ptm_lib_decode_header(csv, &msglen, &ver, &type, &cmd_id,
client_name);
csv_clean(csv);
csv_free(csv);
if (rc < 0) {
/* could not decode the CSV - maybe its legacy cmd?
* get the entire cmd from the socket and see if we can process
* it
*/
if (len == PTMLIB_MSG_HDR_LEN) {
len += _ptm_lib_read_ptm_socket(
fd, (inbuf + PTMLIB_MSG_HDR_LEN),
inlen - PTMLIB_MSG_HDR_LEN);
if (len <= 0)
return (len);
}
inbuf[len] = '\0';
/* we only support the get-status cmd */
if (strcmp(inbuf, PTMLIB_CMD_GET_STATUS)) {
DLOG("unsupported legacy cmd %s\n", inbuf);
return (-1);
}
/* internally create a csv-style cmd */
ptm_lib_init_msg(hdl, 0, PTMLIB_MSG_TYPE_CMD, NULL,
(void *)&p_ctxt);
if (!p_ctxt) {
DLOG("couldnt allocate context\n");
return (-1);
}
ptm_lib_append_msg(hdl, p_ctxt, "cmd", PTMLIB_CMD_GET_STATUS);
} else {
if (msglen > inlen) {
DLOG("msglen [%d] > inlen [%d]\n", msglen, inlen);
return -1;
}
/* read the rest of the msg */
len = _ptm_lib_read_ptm_socket(fd, inbuf, msglen);
if (len <= 0) {
return (len);
}
inbuf[len] = '\0';
csv = csv_init(NULL, NULL, PTMLIB_MSG_SZ);
if (!csv) {
ERRLOG("Cannot allocate csv for msg\n");
return -1;
}
csv_decode(csv, inbuf);
p_ctxt = calloc(1, sizeof(*p_ctxt));
if (!p_ctxt) {
ERRLOG("%s: Could not allocate context \n",
__FUNCTION__);
csv_clean(csv);
csv_free(csv);
return -1;
}
p_ctxt->csv = csv;
p_ctxt->cmd_id = cmd_id;
p_ctxt->type = type;
}
switch (p_ctxt->type) {
case PTMLIB_MSG_TYPE_NOTIFICATION:
if (hdl->notify_cb)
hdl->notify_cb(arg, p_ctxt);
break;
case PTMLIB_MSG_TYPE_CMD:
if (hdl->cmd_cb)
hdl->cmd_cb(arg, p_ctxt);
break;
case PTMLIB_MSG_TYPE_RESPONSE:
if (hdl->response_cb)
hdl->response_cb(arg, p_ctxt);
break;
default:
return -1;
}
csv_clean(p_ctxt->csv);
csv_free(p_ctxt->csv);
free(p_ctxt);
return len;
}
static void add_generic_csv(meta_parameters *meta, const char *csv_file,
int auto_close_polys)
{
int num_meta_cols, num_data_cols;
csv_meta_column_t *meta_column_info;
csv_data_column_t *data_column_info;
if (!meta_supports_meta_get_latLon(meta)) {
asfPrintStatus("No geolocation info - can't add CSV: %s\n", csv_file);
return;
}
asfPrintStatus("Adding: %s\n", csv_file);
if (!fileExists(csv_file)) {
asfPrintWarning("File not found: %s\n", csv_file);
return;
}
FILE *ifp = csv_open(csv_file,
&num_meta_cols, &meta_column_info,
&num_data_cols, &data_column_info);
// csv_open() returns NULL if the file can't be processed
if (!ifp)
return;
// this is just for debugging
//csv_info(num_meta_cols, meta_column_info, num_data_cols, data_column_info);
// start line counter at 1 (header line is not part of this loop)
int i,line_num=1;
int num = 0;
char line[1024];
while (fgets(line, 1023, ifp)) {
++line_num;
char **column_data;
double *lats, *lons;
int ok = csv_line_parse(line, line_num,
num_meta_cols, meta_column_info,
num_data_cols, data_column_info,
&column_data, &lats, &lons);
// csv_line_parse() will return FALSE when the line is invalid
if (!ok)
continue;
++num;
csv_free(num_meta_cols, column_data, lats, lons);
}
FCLOSE(ifp);
FREE(meta_column_info);
meta_column_info = NULL;
FREE(data_column_info);
data_column_info = NULL;
// Use line_num-1 so we do not count the header line
asfPrintStatus("File had %d line%s, %d valid line%s.\n",
line_num-1, line_num-1==1?"":"s", num, num==1?"":"s");
// free pre-existing loaded shapes
if (g_shapes)
free_shapes();
// set up global array of shapes
num_shapes = num;
g_shapes = MALLOC(sizeof(Shape*)*num_shapes);
for (i=0; i<num_shapes; ++i)
g_shapes[i] = NULL;
// now open file again, this time we will actually process!
ifp = csv_open(csv_file, &num_meta_cols, &meta_column_info,
&num_data_cols, &data_column_info);
assert(ifp);
line_num = 1;
num = 0;
while (fgets(line, 1023, ifp)) {
++line_num;
char **column_data;
double *lats, *lons;
int ok = csv_line_parse(line, line_num,
num_meta_cols, meta_column_info,
num_data_cols, data_column_info,
&column_data, &lats, &lons);
// csv_line_parse() will return FALSE when the line is invalid
if (!ok)
continue;
// dealing with metadata
Shape *s = MALLOC(sizeof(Shape));
s->num_meta_cols = num_meta_cols;
s->meta_cols = meta_column_info;
s->meta_info = MALLOC(sizeof(char*)*num_meta_cols);
for (i=0; i<num_meta_cols; ++i)
s->meta_info[i] = STRDUP(column_data[i]);
// dealing with data
s->num_points = num_data_cols;
if (auto_close_polys)
s->num_points++;
s->lines = MALLOC(sizeof(double)*s->num_points);
s->samps = MALLOC(sizeof(double)*s->num_points);
for (i=0; i<num_data_cols; ++i) {
double line, samp;
meta_get_lineSamp(meta, lats[i], lons[i], 0, &line, &samp);
s->lines[i] = line;
s->samps[i] = samp;
}
if (auto_close_polys) {
s->lines[num_data_cols] = s->lines[0];
s->samps[num_data_cols] = s->samps[0];
}
// display info
s->color_code = num%27 + 10;
s->marker_code = 1; // square
g_shapes[num] = s;
++num;
csv_free(num_meta_cols, column_data, lats, lons);
}
FCLOSE(ifp);
FREE(data_column_info);
// do not free meta_column_info -- pointed to by g_shape now
}
std::shared_ptr<storage::AbstractTable> RawTableLoader::load(std::shared_ptr<storage::AbstractTable> in,
const storage::compound_metadata_list *ml,
const Loader::params &args) {
csv::params params;
if (detectHeader(args.getBasePath() + _filename)) params.setLineStart(5);
// Create the result table
storage::metadata_vec_t v(in->columnCount());
for(size_t i=0; i < in->columnCount(); ++i) {
v[i] = in->metadataAt(i);
}
auto result = std::make_shared<storage::RawTable>(v);
// CSV Parsing
std::ifstream file(args.getBasePath() + _filename, std::ios::binary);
if (!file || file.bad()) {
throw csv::ParserError("CSV file '" + _filename + "' does not exist");
}
struct csv_parser parser;
if (!csv_init(&parser, 0)) {
csv_set_opts(&parser, CSV_APPEND_NULL);
csv_set_delim(&parser, params.getDelimiter());
// If there is a header in the file, we will ignore it
std::string line;
int line_start = params.getLineStart();
if (line_start != 1) {
while (line_start > 1) {
std::getline(file, line);
--line_start;
}
}
// Prepare cb data handler
struct raw_table_cb_data data(v);
data.table = result;
const size_t block_size = 16 * 1024;
char rdbuf [block_size];
while (file.read(rdbuf, block_size).good()) {
auto extracted = file.gcount();
if (extracted == 0)
break;
if (csv_parse(&parser,
rdbuf,
extracted,
(field_cb_t) raw_table_cb_per_field,
(line_cb_t) raw_table_cb_per_line,
(void*) &data) != (size_t) extracted) {
throw csv::ParserError(csv_strerror(csv_error(&parser)));
}
}
// Parse the rest
if (csv_parse(&parser,
rdbuf,
file.gcount(),
(field_cb_t) raw_table_cb_per_field,
(line_cb_t) raw_table_cb_per_line,
(void*) &data) != (size_t) file.gcount()) {
throw csv::ParserError(csv_strerror(csv_error(&parser)));
}
csv_fini(&parser,
(field_cb_t) raw_table_cb_per_field,
(line_cb_t) raw_table_cb_per_line,
(void*) &data);
}
csv_free(&parser);
return result;
}
readstat_error_t readstat_parse_csv(readstat_parser_t *parser, const char *path, const char *jsonpath, struct csv_metadata* md, void *user_ctx) {
readstat_error_t retval = READSTAT_OK;
readstat_io_t *io = parser->io;
size_t file_size = 0;
size_t bytes_read;
struct csv_parser csvparser;
struct csv_parser *p = &csvparser;
char buf[BUFSIZ];
size_t* column_width = md->column_width;
md->pass = column_width ? 2 : 1;
md->open_row = 0;
md->columns = 0;
md->_rows = md->rows;
md->rows = 0;
md->parser = parser;
md->user_ctx = user_ctx;
md->json_md = NULL;
if ((md->json_md = get_json_metadata(jsonpath)) == NULL) {
fprintf(stderr, "Could not get JSON metadata\n");
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
if (io->open(path, io->io_ctx) == -1) {
retval = READSTAT_ERROR_OPEN;
goto cleanup;
}
file_size = io->seek(0, READSTAT_SEEK_END, io->io_ctx);
if (file_size == -1) {
retval = READSTAT_ERROR_SEEK;
goto cleanup;
}
if (io->seek(0, READSTAT_SEEK_SET, io->io_ctx) == -1) {
retval = READSTAT_ERROR_SEEK;
goto cleanup;
}
if (csv_init(p, CSV_APPEND_NULL) != 0)
{
retval = READSTAT_ERROR_OPEN;
goto cleanup;
}
unsigned char sep = get_separator(md->json_md);
csv_set_delim(p, sep);
while ((bytes_read = io->read(buf, sizeof(buf), io->io_ctx)) > 0)
{
if (csv_parse(p, buf, bytes_read, csv_metadata_cell, csv_metadata_row, md) != bytes_read)
{
fprintf(stderr, "Error while parsing file: %s\n", csv_strerror(csv_error(p)));
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
}
csv_fini(p, csv_metadata_cell, csv_metadata_row, md);
if (!md->open_row) {
md->rows--;
}
if (parser->info_handler && md->pass == 1) {
parser->info_handler(md->rows, md->_columns, user_ctx);
}
cleanup:
if (md->variables) {
free(md->variables);
md->variables = NULL;
}
if (md->is_date) {
free(md->is_date);
md->is_date = NULL;
}
if (md->json_md) {
free_json_metadata(md->json_md);
md->json_md = NULL;
}
csv_free(p);
io->close(io->io_ctx);
return retval;
}
int main(int argc, char *argv[])
{
//check arguments
if(argc < 2 || argc > 3)
{
printf("evoscan_logworks_conv <version %s>\n", version);
printf("[email protected], @640774n6\n\n");
printf("usage: evoscan_logworks_conv <input csv path> <output dif path (optional)>\n");
return 1;
}
char *input_path = argv[1];
char *generated_output_path = create_output_path(input_path);
char *output_path = (argc == 3) ? argv[2] : generated_output_path;
if(!strcmp(input_path, output_path))
{
printf("error: input and output path must be different\n");
free(generated_output_path);
return 1;
}
//open file pointers
FILE *input_file = fopen(input_path, "rb");
if(!input_file)
{
printf("error: failed to open input @ %s\n", input_path);
free(generated_output_path);
return 1;
}
FILE *output_file = fopen(output_path, "wb");
if(!output_path)
{
printf("error: failed to open output @ %s\n", output_path);
free(generated_output_path);
fclose(input_file);
return 1;
}
FILE *tmp_file = tmpfile();
if(!tmp_file)
{
printf("error: failed to open tmp file\n");
free(generated_output_path);
fclose(input_file);
fclose(output_file);
return 1;
}
//initialize variables
fields = NULL;
field_count = 0;
used_fields = NULL;
used_fields_min = NULL;
used_fields_max = NULL;
used_field_count = 0;
total_sample_time = 0.0;
total_sample_count = 0;
row_sample_count = 0;
csv_col = 0;
csv_row = 0;
//create csv parser
struct csv_parser parser;
unsigned char options = (CSV_APPEND_NULL | CSV_EMPTY_IS_NULL);
csv_init(&parser, options);
//main parse loop
size_t length = 0;
char buffer[1024];
while((length = fread(buffer, 1, 1024, input_file)) > 0)
{
//parse csv and handle with callbacks
if(csv_parse(&parser, buffer, length, csv_process_col, csv_process_row, tmp_file) != length)
{
printf("error: failed to read from input @ %s\n", input_path);
free(generated_output_path);
fclose(input_file);
fclose(output_file);
fclose(tmp_file);
csv_free(&parser);
remove(output_path);
return 1;
}
}
//write output header
fprintf(output_file, "TABLE\r\n0,1\r\n\"EXCEL\"\r\n");
fprintf(output_file, "VECTORS\r\n0,%d\r\n\"\"\r\n", (total_sample_count + 13));
fprintf(output_file, "TUPLES\r\n0,%d\r\n\"\"\r\n", (used_field_count + 1));
fprintf(output_file, "DATA\r\n0,0\r\n\"\"\r\n");
fprintf(output_file, "-1,0\r\nBOT\r\n");
fprintf(output_file, "1,0\r\n\"Input Description\"\r\n");
for(int i = 0; i < used_field_count; i++)
{ fprintf(output_file, "1,0\r\n\"\"\r\n"); }
fprintf(output_file, "-1,0\r\nBOT\r\n");
fprintf(output_file, "1,0\r\n\"Stochiometric:\"\r\n");
for(int i = 0; i < used_field_count; i++)
{ fprintf(output_file, "1,0\r\n\"\"\r\n"); }
fprintf(output_file, "-1,0\r\nBOT\r\n");
fprintf(output_file, "1,0\r\n\"From Device:\"\r\n");
for(int i = 0; i < used_field_count; i++)
{ fprintf(output_file, "1,0\r\n\"(EVOSCAN%d)\"\r\n", (i + 1)); }
fprintf(output_file, "-1,0\r\nBOT\r\n");
fprintf(output_file, "1,0\r\n\"Name:\"\r\n");
for(int i = 0; i < used_field_count; i++)
{ fprintf(output_file, "1,0\r\n\"%s\"\r\n", used_fields[i]); }
fprintf(output_file, "-1,0\r\nBOT\r\n");
fprintf(output_file, "1,0\r\n\"Unit:\"\r\n");
for(int i = 0; i < used_field_count; i++)
{ fprintf(output_file, "1,0\r\n\"%.3s\"\r\n", used_fields[i]); }
fprintf(output_file, "-1,0\r\nBOT\r\n");
fprintf(output_file, "1,0\r\n\"Range:\"\r\n");
for(int i = 0; i < used_field_count; i++)
{ fprintf(output_file, "0,%d\r\nV\r\n", used_fields_min[i]); }
fprintf(output_file, "-1,0\r\nBOT\r\n");
fprintf(output_file, "1,0\r\n\"equiv(Sample):\"\r\n");
for(int i = 0; i < used_field_count; i++)
{ fprintf(output_file, "0,0\r\nV\r\n"); }
fprintf(output_file, "-1,0\r\nBOT\r\n");
fprintf(output_file, "1,0\r\n\"to:\"\r\n");
for(int i = 0; i < used_field_count; i++)
{ fprintf(output_file, "0,%d\r\nV\r\n", used_fields_max[i] + 1); }
fprintf(output_file, "-1,0\r\nBOT\r\n");
fprintf(output_file, "1,0\r\n\"equiv(Sample):\"\r\n");
for(int i = 0; i < used_field_count; i++)
{ fprintf(output_file, "0,4096\r\nV\r\n"); }
fprintf(output_file, "-1,0\r\nBOT\r\n");
fprintf(output_file, "1,0\r\n\"Color:\"\r\n");
for(int i = 0; i < used_field_count; i++)
{ fprintf(output_file, "0,%d\r\nV\r\n", color_for_index(i)); }
fprintf(output_file, "-1,0\r\nBOT\r\n");
fprintf(output_file, "1,0\r\n\"-End-\"\r\n");
for(int i = 0; i < used_field_count; i++)
{ fprintf(output_file, "1,0\r\n\"\"\r\n"); }
fprintf(output_file, "-1,0\r\nBOT\r\n");
fprintf(output_file, "1,0\r\n\"Session 1\"\r\n");
for(int i = 0; i < used_field_count; i++)
{ fprintf(output_file, "1,0\r\n\"\"\r\n"); }
fprintf(output_file, "-1,0\r\nBOT\r\n");
fprintf(output_file, "1,0\r\n\"Time(sec)\"\r\n");
for(int i = 0; i < used_field_count; i++)
{ fprintf(output_file, "1,0\r\n\"%s (%.3s)\"\r\n", used_fields[i], used_fields[i]); }
//append tmp to the output
fseek(tmp_file, 0, SEEK_SET);
while((length = fread(buffer, 1, 1024, tmp_file)) > 0)
{ fwrite(buffer, sizeof(char), length, output_file); }
//write footer
fprintf(output_file, "-1,0\r\nEOD\r\n");
//free generated output path
free(generated_output_path);
//free fields
for(int i = 0; i < field_count; i++)
{ free(fields[i]); }
free(fields);
free(used_fields);
free(used_fields_min);
free(used_fields_max);
//close file pointers
fclose(input_file);
fclose(output_file);
fclose(tmp_file);
//free parser
csv_free(&parser);
return 0;
}
