struct Block * multiply(struct Block * block, struct Block * opt)
{
const char * column_name = get_attribute_value_as_string(opt, "column_name");
double multiple = get_attribute_value_as_double(opt, "multiple");
int column_id = get_column_id_by_name_or_exit(block, (char*)column_name);
if (column_id == -1)
{
fprintf(stderr, "%s() called on '%s' field, wasn't found\n", __func__, column_name);
return block;
}
struct Column * column = get_column(block, column_id);
if (column->type != TYPE_FLOAT)
{
fprintf(stderr, "%s() called on '%s' field, which is of type '%s' (only supports floating point)\n", __func__, column_name, get_type_name(column->type, column->bsize));
return block;
}
char temp[100];
snprintf(temp, sizeof(temp), "multiply(column:\"%s\" by %lf)", column_name, multiple);
block = add_string_attribute(block, "command", temp);
int i;
for (i = 0 ; i < block->num_rows ; i++)
set_cell_from_double(block, i, column_id, get_cell_as_double(block, i, column_id) * multiple);
return block;
}
struct Block * normalize(struct Block * block, struct Block * opt)
{
const char * column_name = get_attribute_value_as_string(opt, "column_name");
double multiple = 1;
struct Attribute * multiple_attr = get_attribute_by_name(opt, "multiple");
if (multiple_attr != NULL && multiple_attr->type == TYPE_FLOAT)
{
if (multiple_attr->value_length == 4) multiple = *(float*)attribute_get_value(multiple_attr);
if (multiple_attr->value_length == 8) multiple = *(double*)attribute_get_value(multiple_attr);
}
int column_id = get_column_id_by_name_or_exit(block, (char*)column_name);
if (column_id == -1)
{
fprintf(stderr, "normalize called on '%s' field, wasn't found\n", column_name);
return block;
}
struct Column * column = get_column(block, column_id);
if (column->type != TYPE_FLOAT)
{
fprintf(stderr, "normalize called on '%s' field, which is of type '%s' (only supports floating point)\n", column_name, get_type_name(column->type, column->bsize));
return block;
}
char temp[100];
snprintf(temp, sizeof(temp), "normalize(column:\"%s\")", column_name);
block = add_string_attribute(block, "command", temp);
double min = DBL_MAX;
double max = -DBL_MAX;
int32_t i;
for (i = 0 ; i < block->num_rows ; i++)
{
double value = get_cell_as_double(block, i, column_id);
if (value < min) min = value;
if (value > max) max = value;
}
for (i = 0 ; i < block->num_rows ; i++)
{
double value = get_cell_as_double(block, i, column_id);
value = (value-min) / (max-min) * multiple;
set_cell_from_double(block, i, column_id, value);
}
return block;
}
/**
* Add a geometry to the shapefile.
*/
bool add(Osmium::Geometry::Geometry* geometry, ///< the geometry
v8::Local<v8::Object> attributes) { ///< a %Javascript object (hash) with the attributes
try {
add_geometry(geometry->create_shp_object());
} catch (Osmium::Exception::IllegalGeometry) {
return false;
}
int ok = 0;
for (size_t n=0; n < m_fields.size(); n++) {
v8::Local<v8::String> key = v8::String::New(m_fields[n].name().c_str());
if (attributes->HasRealNamedProperty(key)) {
v8::Local<v8::Value> value = attributes->GetRealNamedProperty(key);
if (value->IsUndefined() || value->IsNull()) {
DBFWriteNULLAttribute(m_dbf_handle, m_current_shape, n);
} else {
switch (m_fields[n].type()) {
case FTString:
ok = add_string_attribute(n, value);
break;
case FTInteger:
ok = DBFWriteIntegerAttribute(m_dbf_handle, m_current_shape, n, value->Int32Value());
break;
case FTDouble:
throw std::runtime_error("fields of type double not implemented");
break;
case FTLogical:
ok = add_logical_attribute(n, value);
break;
default:
ok = 0; // should never be here
break;
}
if (!ok) {
std::string errmsg("failed to add attribute '");
errmsg += m_fields[n].name();
errmsg += "'\n";
throw std::runtime_error(errmsg);
}
}
} else {
DBFWriteNULLAttribute(m_dbf_handle, m_current_shape, n);
}
}
return true;
}
/**
* Add a geometry to the shapefile.
*
* @param shapefile shapefile the geometry is being added to
* @param geometry geometry that should be added to the shapefile
* @param attributes a %Javascript object (hash) with the attributes
*/
static bool add(Osmium::Export::Shapefile* shapefile,
Osmium::Geometry::Geometry* geometry,
v8::Local<v8::Object> attributes) {
try {
shapefile->add_geometry(Osmium::Geometry::create_shp_object(*geometry));
} catch (Osmium::Geometry::IllegalGeometry) {
return false;
}
for (size_t n=0; n < shapefile->fields().size(); ++n) {
v8::Local<v8::String> key = v8::String::New(shapefile->field(n).name().c_str());
if (attributes->HasRealNamedProperty(key)) {
v8::Local<v8::Value> value = attributes->GetRealNamedProperty(key);
if (value->IsUndefined() || value->IsNull()) {
shapefile->add_attribute(n);
} else {
switch (shapefile->field(n).type()) {
case FTString:
add_string_attribute(shapefile, n, value);
break;
case FTInteger:
shapefile->add_attribute(n, value->Int32Value());
break;
case FTDouble:
throw std::runtime_error("fields of type double not implemented");
break;
case FTLogical:
add_logical_attribute(shapefile, n, value);
break;
default:
// should never be here
break;
}
}
} else {
shapefile->add_attribute(n);
}
}
return true;
}
int main(int argc, char ** argv)
{
static char filename[1000] = "";
static int output_header = 1;
int c;
while (1)
{
static struct option long_options[] = {
{"filename", required_argument, 0, 'f'},
{0, 0, 0, 0}
};
int option_index = 0;
c = getopt_long(argc, argv, "f:", long_options, &option_index);
if (c == -1) break;
switch (c)
{
case 0: break;
case 'f': strncpy(filename, optarg, sizeof(filename)); break;
default: abort();
}
}
if (filename[0] == 0 && argc == 2 && argv[1] != NULL)
strncpy(filename, argv[1], sizeof(filename));
if (filename[0] == 0) { fprintf(stderr, "ERROR %s: filename not provided\n", argv[0]); return EXIT_FAILURE; }
FILE * fp = fopen(filename, "r");
if (fgetc(fp) != 0xEF || fgetc(fp) != 0xBB || fgetc(fp) != 0xBF) rewind(fp); // skip UTF-8 BOM if it's found
if (fp != NULL)
{
struct Block * block = new_block();
block = add_command(block, argc, argv);
block = add_string_attribute(block, "source csv file", filename);
char header[500] = "";
if (fgets(header, sizeof(header), fp) == NULL)
{
fprintf(stderr, "ERROR: could not read header from '%s'\n", filename);
exit(1);
}
int length = 0;
char * line_ptr = header;
char * field;
while ((field = strsep(&line_ptr, ",")) != NULL)
{
length = strlen(field)-1;
if (field[length] == '\r' || field[length] == '\n') field[length] = 0;
length = strlen(field)-1;
if (field[length] == '\r' || field[length] == '\n') field[length] = 0;
block = add_string_column_with_length(block, field, 1);
}
int row_id = 0;
char row[500] = "";
while (fgets(row, sizeof(row), fp) != NULL)
{
length = strlen(row)-1;
if (row[length] == '\r' || row[length] == '\n') row[length] = 0;
length = strlen(row)-1;
if (row[length] == '\r' || row[length] == '\n') row[length] = 0;
//fprintf(stderr, "row: \"%s\"\n", row);
if (length == 0) continue;
block = add_row(block);
int column_id = 0;
line_ptr = row;
while ((field = strsep(&line_ptr, ",")) != NULL)
{
length = (line_ptr == 0) ? strlen(field) : line_ptr - field - 1;
if (field[0] == '"')
{
int count = 0;
char * start_of_field = field;
int length = strlen(field);
while (field[strlen(field)-1] != '"' && count < 100)
{
//fprintf(stderr, "last char = %d\n", field[strlen(field)-1]);
//fprintf(stderr, "count = %d\n", count);
//fprintf(stderr, "line_ptr = '%s'\n", line_ptr);
//fprintf(stderr, "field = '%s'\n", field);
length = strlen(field);
//fprintf(stderr, "length = '%d'\n", length);
field[length] = ',';
line_ptr ++;
//fprintf(stderr, "new field = '%s'\n", field);
//fprintf(stderr, "new line_ptr = '%s'\n", line_ptr);
strsep(&line_ptr, ",");
count++;
}
if (count == 100)
{
fprintf(stderr, "major problem reading file in read_csv_fast, perhaps use read_csv instead\n");
}
field = start_of_field;
if (field[0] == '"' && field[strlen(field)-1] == '"')
{
field[strlen(field)-1] = 0;
field++;
}
}
//fprintf(stderr, " cell: \"%s\" (%d)\n", field, length);
struct Column * column = get_column(block, column_id);
if (length >= column->bsize)
{
//fprintf(stderr, "row %d, column %d becomes %d big (was %d)\n", row_id, column_id, memory_pad(length+1,4), column->bsize);
block = set_string_column_length(block, column_id, length);
}
set_cell_from_string(block, row_id, column_id, field);
char * cell = get_cell(block, row_id, column_id);
//fprintf(stderr, " cell: \"%s\" (%d)\n", cell, length);
//cell[memory_pad(length+1,4)-1] = 0;
//cell[length] = 0;
column_id++;
}
row_id++;
//if (row_id > 10)
// break;
}
write_block(stdout, block);
free_block(block);
fclose(fp);
}
else
{
fprintf(stderr, "'%s' doesn't exist.\n", filename);
}
return EXIT_SUCCESS;
}
int main(int argc, char ** argv)
{
if (stdout_is_piped()) // other wise you don't see the seg fault
setup_segfault_handling(argv);
assert_stdin_is_piped();
assert_stdout_is_piped();
//assert_stdin_or_out_is_piped();
static char remove_columns_all[1000] = "";
static char int_columns_all[1000] = "";
static char intfromtime_columns_all[1000] = "";
static char float_columns_all[1000] = "";
static int output_header = 1;
static int debug = 0;
int c;
while (1)
{
static struct option long_options[] = {
{"remove", required_argument, 0, 'r'},
{"makeint", required_argument, 0, 'i'},
{"makeintfromtime", required_argument, 0, 't'},
{"makefloat", required_argument, 0, 'f'},
//{"add", no_argument, &output_header, 1},
{"debug", no_argument, &debug, 1},
{0, 0, 0, 0}
};
int option_index = 0;
c = getopt_long(argc, argv, "r:i:", long_options, &option_index);
if (c == -1) break;
switch (c)
{
case 0: break;
case 'r': strncpy(remove_columns_all, optarg, sizeof(remove_columns_all)); break;
case 'i': strncpy(int_columns_all, optarg, sizeof(int_columns_all)); break;
case 't': strncpy(intfromtime_columns_all, optarg, sizeof(intfromtime_columns_all)); break;
case 'f': strncpy(float_columns_all, optarg, sizeof(float_columns_all)); break;
default: abort();
}
}
char remove_columns_copy[1000] = "";
strncpy(remove_columns_copy, remove_columns_all, 1000);
char int_columns_copy[1000] = "";
strncpy(int_columns_copy, int_columns_all, 1000);
char intfromtime_columns_copy[1000] = "";
strncpy(intfromtime_columns_copy, intfromtime_columns_all, 1000);
char float_columns_copy[1000] = "";
strncpy(float_columns_copy, float_columns_all, 1000);
int num_remove_columns = 0;
char ** remove_columns = NULL;
char * pch = strtok(remove_columns_all, ",");
while (pch != NULL)
{
num_remove_columns++;
remove_columns = realloc(remove_columns, sizeof(char*)*num_remove_columns);
remove_columns[num_remove_columns-1] = pch;
pch = strtok(NULL, ",");
}
int num_int_columns = 0;
char ** int_columns = NULL;
pch = strtok(int_columns_all, ",");
while (pch != NULL)
{
num_int_columns++;
int_columns = realloc(int_columns, sizeof(char*)*num_int_columns);
int_columns[num_int_columns-1] = pch;
pch = strtok(NULL, ",");
}
int num_intfromtime_columns = 0;
char ** intfromtime_columns = NULL;
pch = strtok(intfromtime_columns_all, ",");
while (pch != NULL)
{
num_intfromtime_columns++;
intfromtime_columns = realloc(intfromtime_columns, sizeof(char*)*num_intfromtime_columns);
intfromtime_columns[num_intfromtime_columns-1] = pch;
pch = strtok(NULL, ",");
}
int num_float_columns = 0;
char ** float_columns = NULL;
pch = strtok(float_columns_all, ",");
while (pch != NULL)
{
num_float_columns++;
float_columns = realloc(float_columns, sizeof(char*)*num_float_columns);
float_columns[num_float_columns-1] = pch;
pch = strtok(NULL, ",");
}
int i,j;
struct Block * block = NULL;
while ((block = read_block(stdin)))
{
struct Block * newblock = new_block();
newblock = copy_all_attributes(newblock, block);
if (remove_columns_copy[0] != 0)
newblock = add_string_attribute(newblock, "remove attributes", remove_columns_copy);
if (int_columns_copy[0] != 0)
newblock = add_string_attribute(newblock, "convert columns to int", int_columns_copy);
if (intfromtime_columns_copy[0] != 0)
newblock = add_string_attribute(newblock, "convert columns to time", intfromtime_columns_copy);
if (float_columns_copy[0] != 0)
newblock = add_string_attribute(newblock, "convert columns to float", float_columns_copy);
int num_remove_column_ids = 0;
int * remove_column_ids = NULL;
for (i = 0 ; i < block->num_columns ; i++)
{
struct Column * col = get_column(block, i);
for (j = 0 ; j < num_remove_columns ; j++)
{
// fprintf(stderr, "%s(%d(%d)) vs %s(%d) = %d\n",
// column_get_name(col), col->name_length, strlen(column_get_name(col)),
// remove_columns[j], strlen(remove_columns[j]),
// strcmp(column_get_name(col), remove_columns[j]));
if (strcmp(column_get_name(col), remove_columns[j])==0) break;
}
if (j == num_remove_columns) // not a column to be removed
{
//fprintf(stderr, "%d: %s (%d) - do not remove\n", i, column_get_name(col), strlen(column_get_name(col)));
for (j = 0 ; j < num_int_columns ; j++)
if (strcmp(column_get_name(col), int_columns[j])==0) break;
int l = 0;
for (l = 0 ; l < num_intfromtime_columns ; l++)
if (strcmp(column_get_name(col), intfromtime_columns[l])==0) break;
int k = 0;
for (k = 0 ; k < num_float_columns ; k++)
if (strcmp(column_get_name(col), float_columns[k])==0) break;
if (j == num_int_columns && k == num_float_columns && l == num_intfromtime_columns)
{
num_remove_column_ids++;
remove_column_ids = realloc(remove_column_ids, sizeof(int)*num_remove_column_ids);
remove_column_ids[num_remove_column_ids-1] = i;
newblock = _add_column(newblock, col->type, col->bsize, column_get_name(col));
}
}
}
int num_int_column_ids = 0;
int * int_column_ids = NULL;
for (i = 0 ; i < block->num_columns ; i++)
{
struct Column * col = get_column(block, i);
for (j = 0 ; j < num_int_columns ; j++)
if (strcmp(column_get_name(col), int_columns[j])==0) break;
if (j != num_int_columns)
{
num_int_column_ids++;
int_column_ids = realloc(int_column_ids, sizeof(int)*num_int_column_ids);
int_column_ids[num_int_column_ids-1] = i;
newblock = add_int32_column(newblock, column_get_name(col));
}
}
int num_intfromtime_column_ids = 0;
int * intfromtime_column_ids = NULL;
for (i = 0 ; i < block->num_columns ; i++)
{
struct Column * col = get_column(block, i);
for (j = 0 ; j < num_intfromtime_columns ; j++)
if (strcmp(column_get_name(col), intfromtime_columns[j])==0) break;
if (j != num_intfromtime_columns)
{
num_intfromtime_column_ids++;
intfromtime_column_ids = realloc(intfromtime_column_ids, sizeof(int)*num_intfromtime_column_ids);
intfromtime_column_ids[num_intfromtime_column_ids-1] = i;
newblock = add_int32_column(newblock, column_get_name(col));
}
}
int num_float_column_ids = 0;
int * float_column_ids = NULL;
for (i = 0 ; i < block->num_columns ; i++)
{
struct Column * col = get_column(block, i);
for (j = 0 ; j < num_float_columns ; j++)
if (strcmp(column_get_name(col), float_columns[j])==0) break;
if (j != num_float_columns)
{
num_float_column_ids++;
float_column_ids = realloc(float_column_ids, sizeof(int)*num_float_column_ids);
float_column_ids[num_float_column_ids-1] = i;
newblock = add_double_column(newblock, column_get_name(col));
}
}
newblock = set_num_rows(newblock, block->num_rows);
for (i = 0 ; i < newblock->num_rows ; i++)
{
for (j = 0 ; j < num_remove_column_ids ; j++)
{
void * dst = get_cell(newblock, i, j);
void * src = get_cell(block, i, remove_column_ids[j]);
struct Column * col = get_column(block, remove_column_ids[j]);
memcpy(dst, src, col->bsize);
}
for (j = 0 ; j < num_int_column_ids ; j++)
{
set_cell_from_int32(newblock, i, j + num_remove_column_ids, get_cell_as_int32(block, i, int_column_ids[j]));
}
for (j = 0 ; j < num_intfromtime_column_ids ; j++)
{
char * cell = get_cell(block, i, intfromtime_column_ids[j]);
int len = strlen(cell);
if (len >= 7)
{
int32_t t = atoi(&cell[len-2]) + atoi(&cell[len-5])*60 + atoi(cell)*60*60;
set_cell_from_int32(newblock, i, j + num_remove_column_ids + num_int_column_ids, t);
}
else
{
fprintf(stderr, "ABORTING: %s expecting time to be in #:##:## format. But length invalid. (%d)\n", __func__, len);
exit(1);
}
}
for (j = 0 ; j < num_float_column_ids ; j++)
{
set_cell_from_double(newblock, i, j + num_remove_column_ids + num_int_column_ids + num_intfromtime_column_ids, get_cell_as_double(block, i, float_column_ids[j]));
}
}
free(remove_column_ids);
free(int_column_ids);
free(float_column_ids);
write_block(stdout, newblock);
free_block(newblock);
free_block(block);
}
}
int main(int argc, char ** argv)
{
if (stdout_is_piped()) // other wise you don't see the seg fault
setup_segfault_handling(argv);
assert_stdin_is_piped();
assert_stdout_is_piped();
//assert_stdin_or_out_is_piped();
static char remove_attributes_all[1000] = "";
static char int_attributes_all[1000] = "";
static int debug = 0;
struct Params * params = NULL;
params = add_string_param(params, "remove", 'r', remove_attributes_all, 0);
params = add_string_param(params, "makeint", 'i', int_attributes_all, 0);
params = add_flag_param(params, "debug", 'd', &debug, 0);
char remove_attributes_copy[1000] = "";
strncpy(remove_attributes_copy, remove_attributes_all, 1000);
char int_attributes_copy[1000] = "";
strncpy(int_attributes_copy, int_attributes_all, 1000);
int num_remove_attributes = 0;
char ** remove_attributes = NULL;
char * pch = strtok(remove_attributes_all, ",");
while (pch != NULL)
{
num_remove_attributes++;
remove_attributes = realloc(remove_attributes, sizeof(char*)*num_remove_attributes);
remove_attributes[num_remove_attributes-1] = pch;
pch = strtok(NULL, ",");
}
int num_int_attributes = 0;
char ** int_attributes = NULL;
pch = strtok(int_attributes_all, ",");
while (pch != NULL)
{
num_int_attributes++;
int_attributes = realloc(int_attributes, sizeof(char*)*num_int_attributes);
int_attributes[num_int_attributes-1] = pch;
pch = strtok(NULL, ",");
}
int i,j;
struct Block * block = NULL;
while ((block = read_block(stdin)))
{
struct Block * newblock = new_block();
newblock = copy_all_attributes(newblock, block);
if (remove_attributes_copy[0] != 0)
newblock = add_string_attribute(newblock, "remove attributes", remove_attributes_copy);
if (int_attributes_copy[0] != 0)
newblock = add_string_attribute(newblock, "convert attributes to int", int_attributes_copy);
int num_remove_attribute_ids = 0;
int * remove_attribute_ids = NULL;
for (i = 0 ; i < block->num_attributes ; i++)
{
struct Attribute * attr = get_attribute(block, i);
for (j = 0 ; j < num_remove_attributes ; j++)
{
// fprintf(stderr, "%s(%d(%d)) vs %s(%d) = %d\n",
// attribute_get_name(attr), attr->name_length, strlen(attribute_get_name(attr)),
// remove_attributes[j], strlen(remove_attributes[j]),
// strcmp(attribute_get_name(attr), remove_attributes[j]));
if (strcmp(attribute_get_name(attr), remove_attributes[j])==0) break;
}
if (j == num_remove_attributes) // not a attribute to be removed
{
//fprintf(stderr, "%d: %s (%d) - do not remove\n", i, attribute_get_name(attr), strlen(attribute_get_name(attr)));
for (j = 0 ; j < num_int_attributes ; j++)
if (strcmp(attribute_get_name(attr), int_attributes[j])==0) break;
if (j == num_int_attributes)
{
num_remove_attribute_ids++;
remove_attribute_ids = realloc(remove_attribute_ids, sizeof(int)*num_remove_attribute_ids);
remove_attribute_ids[num_remove_attribute_ids-1] = i;
newblock = _add_attribute(newblock, attr->type, attr->value_length, attribute_get_name(attr), attribute_get_value(attr));
}
}
}
int num_int_attribute_ids = 0;
int * int_attribute_ids = NULL;
for (i = 0 ; i < block->num_attributes ; i++)
{
struct Attribute * attr = get_attribute(block, i);
for (j = 0 ; j < num_int_attributes ; j++)
if (strcmp(attribute_get_name(attr), int_attributes[j])==0) break;
if (j != num_int_attributes)
{
num_int_attribute_ids++;
int_attribute_ids = realloc(int_attribute_ids, sizeof(int)*num_int_attribute_ids);
int_attribute_ids[num_int_attribute_ids-1] = i;
newblock = add_int32_attribute(newblock, attribute_get_name(attr), *(int32_t*)attribute_get_value(attr));
}
}
newblock = set_num_rows(newblock, block->num_rows);
for (i = 0 ; i < newblock->num_rows ; i++)
{
for (j = 0 ; j < num_remove_attribute_ids ; j++)
{
void * dst = get_cell(newblock, i, j);
void * src = get_cell(block, i, remove_attribute_ids[j]);
struct Attribute * attr = get_attribute(block, remove_attribute_ids[j]);
memcpy(dst, src, attr->value_length);
}
for (j = 0 ; j < num_int_attribute_ids ; j++)
{
set_cell_from_int32(newblock, i, j + num_remove_attribute_ids, get_cell_as_int32(block, i, int_attribute_ids[j]));
}
}
free(remove_attribute_ids);
free(int_attribute_ids);
write_block(stdout, newblock);
free_block(newblock);
free_block(block);
}
}
