static int import_csv(const char* path)
{
FILE* fp = fopen(path, "r");
if (fp == NULL) {
fprintf(stderr, "File not found: %s\n", path);
return -1;
}
hashtable_t* datastreams = new_hashtable(NULL);
while (1) {
unsigned long id;
double t, v;
int n = fscanf(fp, "%lu,%lf,%lf\n", &id, &t, &v);
if (n < 3)
break;
datastream_t* d = NULL;
hashtable_lookup(datastreams, id, (void**) &d);
if (d == NULL) {
d = datastream_load(id);
if (d == NULL) {
d = datastream_create(id, 0.5, 3600.0, 0.1);
if (d == NULL) {
fprintf(stderr, "Failed to create the datastream\n");
goto error_recovery;
}
}
hashtable_insert(datastreams, id, d);
}
printf("id=%d, t=%d, v=%f\n", (int) id, (int) t, v);
int err = datastream_insert(d, (timestamp_t) t, (value_t) v);
if (err != 0)
goto error_recovery;
}
fclose(fp);
hashtable_foreach(datastreams, store_datastreams_callback, NULL);
return 0;
error_recovery:
fclose(fp);
hashtable_foreach(datastreams, store_datastreams_callback, NULL);
return -1;
}
obj hashtable_values_to_list( obj table )
{
obj list = NIL_OBJ;
hashtable_foreach( table, &list, accum_value );
return list;
}
obj hashtable_keys_to_list( obj table )
{
obj list = NIL_OBJ;
hashtable_foreach( table, &list, accum_key );
return list;
}
void unbind_all_names_for_node(node_t *n) {
hashtable_t names = n->names;
init_node_names(n);
hashtable_foreach(&names, unbind_on_destroy, NULL);
destroy_hashtable(&names);
}
int
quantize(Pixel *pixelData,
uint32_t nPixels,
uint32_t nQuantPixels,
Pixel **palette,
uint32_t *paletteLength,
uint32_t **quantizedPixels,
int kmeans)
{
PixelList *hl[3];
HashTable *h;
BoxNode *root;
uint32_t i;
uint32_t *qp;
uint32_t nPaletteEntries;
uint32_t *avgDist;
uint32_t **avgDistSortKey;
Pixel *p;
#ifndef NO_OUTPUT
uint32_t timer,timer2;
#endif
#ifndef NO_OUTPUT
timer2=clock();
printf ("create hash table..."); fflush(stdout); timer=clock();
#endif
h=create_pixel_hash(pixelData,nPixels);
#ifndef NO_OUTPUT
printf ("done (%f)\n",(clock()-timer)/(double)CLOCKS_PER_SEC);
#endif
if (!h) {
goto error_0;
}
#ifndef NO_OUTPUT
printf ("create lists from hash table..."); fflush(stdout); timer=clock();
#endif
hl[0]=hl[1]=hl[2]=NULL;
hashtable_foreach(h,hash_to_list,hl);
#ifndef NO_OUTPUT
printf ("done (%f)\n",(clock()-timer)/(double)CLOCKS_PER_SEC);
#endif
if (!hl[0]) {
goto error_1;
}
#ifndef NO_OUTPUT
printf ("mergesort lists..."); fflush(stdout); timer=clock();
#endif
for(i=0;i<3;i++) {
hl[i]=mergesort_pixels(hl[i],i);
}
#ifdef TEST_MERGESORT
if (!test_sorted(hl)) {
printf ("bug in mergesort\n");
goto error_1;
}
#endif
#ifndef NO_OUTPUT
printf ("done (%f)\n",(clock()-timer)/(double)CLOCKS_PER_SEC);
#endif
#ifndef NO_OUTPUT
printf ("median cut..."); fflush(stdout); timer=clock();
#endif
root=median_cut(hl,nPixels,nQuantPixels);
#ifndef NO_OUTPUT
printf ("done (%f)\n",(clock()-timer)/(double)CLOCKS_PER_SEC);
#endif
if (!root) {
goto error_1;
}
nPaletteEntries=0;
#ifndef NO_OUTPUT
printf ("median cut tree to hash table..."); fflush(stdout); timer=clock();
#endif
annotate_hash_table(root,h,&nPaletteEntries);
#ifndef NO_OUTPUT
printf ("done (%f)\n",(clock()-timer)/(double)CLOCKS_PER_SEC);
#endif
#ifndef NO_OUTPUT
printf ("compute palette...\n"); fflush(stdout); timer=clock();
#endif
if (!compute_palette_from_median_cut(pixelData,nPixels,h,&p,nPaletteEntries)) {
goto error_3;
}
#ifndef NO_OUTPUT
printf ("done (%f)\n",(clock()-timer)/(double)CLOCKS_PER_SEC);
#endif
free_box_tree(root);
root=NULL;
/* malloc check ok, using calloc for overflow */
qp=calloc(nPixels, sizeof(uint32_t));
if (!qp) { goto error_4; }
if (nPaletteEntries > UINT32_MAX / nPaletteEntries ) {
goto error_5;
}
/* malloc check ok, using calloc for overflow, check of n*n above */
avgDist=calloc(nPaletteEntries*nPaletteEntries, sizeof(uint32_t));
if (!avgDist) { goto error_5; }
/* malloc check ok, using calloc for overflow, check of n*n above */
avgDistSortKey=calloc(nPaletteEntries*nPaletteEntries, sizeof(uint32_t *));
if (!avgDistSortKey) { goto error_6; }
if (!build_distance_tables(avgDist,avgDistSortKey,p,nPaletteEntries)) {
goto error_7;
}
if (!map_image_pixels_from_median_box(pixelData,nPixels,p,nPaletteEntries,h,avgDist,avgDistSortKey,qp)) {
goto error_7;
}
#ifdef TEST_NEAREST_NEIGHBOUR
#include <math.h>
{
uint32_t bestmatch,bestdist,dist;
HashTable *h2;
printf ("nearest neighbour search (full search)..."); fflush(stdout); timer=clock();
h2=hashtable_new(unshifted_pixel_hash,unshifted_pixel_cmp);
for (i=0;i<nPixels;i++) {
if (hashtable_lookup(h2,pixelData[i],&paletteEntry)) {
bestmatch=paletteEntry;
} else {
bestmatch=0;
bestdist=
_SQR(pixelData[i].c.r-p[0].c.r)+
_SQR(pixelData[i].c.g-p[0].c.g)+
_SQR(pixelData[i].c.b-p[0].c.b);
for (j=1;j<nPaletteEntries;j++) {
dist=
_SQR(pixelData[i].c.r-p[j].c.r)+
_SQR(pixelData[i].c.g-p[j].c.g)+
_SQR(pixelData[i].c.b-p[j].c.b);
if (dist==bestdist && j==qp[i]) {
bestmatch=j;
}
if (dist<bestdist) {
bestdist=dist;
bestmatch=j;
}
}
hashtable_insert(h2,pixelData[i],bestmatch);
}
if (qp[i]!=bestmatch ) {
printf ("discrepancy in matching algorithms pixel %d [%d %d] %f %f\n",
i,qp[i],bestmatch,
sqrt((double)(_SQR(pixelData[i].c.r-p[qp[i]].c.r)+
_SQR(pixelData[i].c.g-p[qp[i]].c.g)+
_SQR(pixelData[i].c.b-p[qp[i]].c.b))),
sqrt((double)(_SQR(pixelData[i].c.r-p[bestmatch].c.r)+
_SQR(pixelData[i].c.g-p[bestmatch].c.g)+
_SQR(pixelData[i].c.b-p[bestmatch].c.b)))
);
}
}
hashtable_free(h2);
}
#endif
#ifndef NO_OUTPUT
printf ("k means...\n"); fflush(stdout); timer=clock();
#endif
if (kmeans) k_means(pixelData,nPixels,p,nPaletteEntries,qp,kmeans-1);
#ifndef NO_OUTPUT
printf ("done (%f)\n",(clock()-timer)/(double)CLOCKS_PER_SEC);
#endif
*quantizedPixels=qp;
*palette=p;
*paletteLength=nPaletteEntries;
#ifndef NO_OUTPUT
printf ("cleanup..."); fflush(stdout); timer=clock();
#endif
if (avgDist) free(avgDist);
if (avgDistSortKey) free(avgDistSortKey);
destroy_pixel_hash(h);
#ifndef NO_OUTPUT
printf ("done (%f)\n",(clock()-timer)/(double)CLOCKS_PER_SEC);
printf ("-----\ntotal time %f\n",(clock()-timer2)/(double)CLOCKS_PER_SEC);
#endif
return 1;
error_7:
if (avgDistSortKey) free(avgDistSortKey);
error_6:
if (avgDist) free(avgDist);
error_5:
if (qp) free(qp);
error_4:
if (p) free(p);
error_3:
if (root) free_box_tree(root);
error_1:
destroy_pixel_hash(h);
error_0:
*quantizedPixels=NULL;
*paletteLength=0;
*palette=NULL;
return 0;
}
void config_save (struct config_t *conf, struct error_t *error)
{
/* for each element in the hash table, call _config_set_key_value() */
hashtable_foreach (conf->hashv, _config_set_key_value, conf);
}
static void setup_replacement_objects( obj rplc_tbl )
{
hi_init_queue_n( &rplc_queue, hashtable_size(rplc_tbl) );
hashtable_foreach( rplc_tbl, (void *)&rplc_queue, init_rplc );
}
