void test_streaming( void ) { mongo conn[1]; gridfs gfs[1]; gridfile gfile[1]; char *medium = (char*)bson_malloc( 2*MEDIUM ); char *small = (char*)bson_malloc( LOWER ); char *buf = (char*)bson_malloc( LARGE ); int n; if( buf == NULL || small == NULL ) { printf( "Failed to allocate" ); exit( 1 ); } srand( (unsigned int)time( NULL ) ); INIT_SOCKETS_FOR_WINDOWS; if ( mongo_client( conn , TEST_SERVER, 27017 ) ) { printf( "failed to connect 3\n" ); exit( 1 ); } fill_buffer_randomly( medium, ( int64_t )2 * MEDIUM ); fill_buffer_randomly( small, ( int64_t )LOWER ); fill_buffer_randomly( buf, ( int64_t )LARGE ); gridfs_init( conn, "test", "fs", gfs ); gridfile_writer_init( gfile, gfs, "medium", "text/html", GRIDFILE_DEFAULT ); gridfile_write_buffer( gfile, medium, MEDIUM ); gridfile_write_buffer( gfile, medium + MEDIUM, MEDIUM ); gridfile_writer_done( gfile ); test_gridfile( gfs, medium, 2 * MEDIUM, "medium", "text/html" ); gridfs_destroy( gfs ); gridfs_init( conn, "test", "fs", gfs ); gridfs_store_buffer( gfs, small, LOWER, "small", "text/html", GRIDFILE_DEFAULT ); test_gridfile( gfs, small, LOWER, "small", "text/html" ); gridfs_destroy( gfs ); gridfs_init( conn, "test", "fs", gfs ); gridfs_remove_filename( gfs, "large" ); gridfile_writer_init( gfile, gfs, "large", "text/html", GRIDFILE_DEFAULT ); for( n=0; n < ( LARGE / 1024 ); n++ ) { gridfile_write_buffer( gfile, buf + ( n * 1024 ), 1024 ); } gridfile_writer_done( gfile ); test_gridfile( gfs, buf, LARGE, "large", "text/html" ); gridfs_destroy( gfs ); mongo_destroy( conn ); free( buf ); free( small ); free( medium ); }
EXPORT void mongo_gridfile_writer_write(struct gridfile_* gf, mxArray* data) { uint64_t size; void* p; int cplx = mxIsComplex(data); if (cplx && mxGetClassID(data) != mxDOUBLE_CLASS) mexErrMsgTxt("GridfileWriter:write - only complex values of type double are supported"); p = calcSize(data, &size); gridfile_write_buffer((gridfile*)gf, (char*)p, size); if (cplx) gridfile_write_buffer((gridfile*)gf, (char*)mxGetPi(data), size); }
void test_large() { mongo_connection conn[1]; mongo_connection_options opts; gridfs gfs[1]; gridfile gfile[1]; FILE *fd; int i, n; char buffer[LARGE]; int64_t filesize = (int64_t)1024 * (int64_t)LARGE; srand(time(NULL)); INIT_SOCKETS_FOR_WINDOWS; strncpy(opts.host, "127.0.0.1", 255); opts.host[254] = '\0'; opts.port = 27017; if (mongo_connect( conn , &opts )) { printf("failed to connect\n"); exit(1); } gridfs_init(conn, "test", "fs", gfs); /* Create a very large file */ fill_buffer_randomly(buffer, (int64_t)LARGE); fd = fopen("bigfile", "w"); for(i=0; i<1024; i++) { fwrite(buffer, 1, LARGE, fd); } fclose(fd); /* Now read the file into GridFS */ gridfs_store_file(gfs, "bigfile", "bigfile", "text/html"); gridfs_find_filename(gfs, "bigfile", gfile); ASSERT( strcmp( gridfile_get_filename( gfile ), "bigfile" ) == 0 ); ASSERT( gridfile_get_contentlength( gfile ) == filesize ); /* Read the file using the streaming interface */ gridfile_writer_init( gfile, gfs, "bigfile-stream", "text/html"); fd = fopen("bigfile", "r"); while((n = fread(buffer, 1, 1024, fd)) != 0) { gridfile_write_buffer(gfile, buffer, n); } gridfile_writer_done( gfile ); gridfs_find_filename(gfs, "bigfile-stream", gfile); ASSERT( strcmp( gridfile_get_filename( gfile ), "bigfile-stream" ) == 0 ); ASSERT( gridfile_get_contentlength( gfile ) == filesize ); gridfs_destroy(gfs); mongo_destroy(conn); }
void test_large() { mongo_connection conn[1]; gridfs gfs[1]; gridfile gfile[1]; FILE *fd; int i, n; char *buffer = malloc( LARGE ); if( buffer == NULL ) { printf("Failed to allocate memory."); exit(1); } uint64_t filesize = (uint64_t)1024 * (uint64_t)LARGE; srand(time(NULL)); INIT_SOCKETS_FOR_WINDOWS; if (mongo_connect( conn, TEST_SERVER, 27017 )){ printf("failed to connect 1\n"); exit(1); } gridfs_init(conn, "test", "fs", gfs); /* Create a very large file */ fill_buffer_randomly(buffer, (uint64_t)LARGE); fd = fopen("bigfile", "w"); for(i=0; i<1024; i++) { fwrite(buffer, 1, LARGE, fd); } fclose(fd); /* Now read the file into GridFS */ gridfs_store_file(gfs, "bigfile", "bigfile", "text/html"); gridfs_find_filename(gfs, "bigfile", gfile); ASSERT( strcmp( gridfile_get_filename( gfile ), "bigfile" ) == 0 ); ASSERT( gridfile_get_contentlength( gfile ) == filesize ); /* Read the file using the streaming interface */ gridfile_writer_init( gfile, gfs, "bigfile-stream", "text/html"); fd = fopen("bigfile", "r"); while((n = fread(buffer, 1, 1024, fd)) != 0) { gridfile_write_buffer(gfile, buffer, n); } gridfile_writer_done( gfile ); gridfs_find_filename(gfs, "bigfile-stream", gfile); ASSERT( strcmp( gridfile_get_filename( gfile ), "bigfile-stream" ) == 0 ); ASSERT( gridfile_get_contentlength( gfile ) == filesize ); gridfs_destroy(gfs); mongo_disconnect(conn); mongo_destroy(conn); }
void test_streaming() { mongo_connection conn[1]; gridfs gfs[1]; gridfile gfile[1]; char *buf = malloc( LARGE ); char *small = malloc( LOWER ); if( buf == NULL || small == NULL ) { printf("Failed to allocate"); exit(1); } int n; srand(time(NULL)); INIT_SOCKETS_FOR_WINDOWS; if (mongo_connect( conn , TEST_SERVER, 27017 )){ printf("failed to connect 3\n"); exit(1); } fill_buffer_randomly(small, (uint64_t)LOWER); fill_buffer_randomly(buf, (uint64_t)LARGE); gridfs_init(conn, "test", "fs", gfs); gridfs_store_buffer(gfs, small, LOWER, "small", "text/html"); test_gridfile(gfs, small, LOWER, "small", "text/html"); gridfs_destroy(gfs); gridfs_init(conn, "test", "fs", gfs); gridfile_writer_init(gfile, gfs, "large", "text/html"); for(n=0; n < (LARGE / 1024); n++) { gridfile_write_buffer(gfile, buf + (n * 1024), 1024); } gridfile_writer_done( gfile ); test_gridfile(gfs, buf, LARGE, "large", "text/html"); gridfs_destroy(gfs); mongo_destroy(conn); free(buf); free(small); }
void test_streaming() { mongo_connection conn[1]; mongo_connection_options opts; gridfs gfs[1]; gridfile gfile[1]; char buf[LARGE]; char small[LOWER]; int n; srand(time(NULL)); INIT_SOCKETS_FOR_WINDOWS; strncpy(opts.host, "127.0.0.1", 255); opts.host[254] = '\0'; opts.port = 27017; if (mongo_connect( conn , &opts )) { printf("failed to connect\n"); exit(1); } fill_buffer_randomly(small, (int64_t)LOWER); fill_buffer_randomly(buf, (int64_t)LARGE); gridfs_init(conn, "test", "fs", gfs); gridfs_store_buffer(gfs, small, LOWER, "small", "text/html"); test_gridfile(gfs, small, LOWER, "small", "text/html"); gridfs_destroy(gfs); gridfs_init(conn, "test", "fs", gfs); gridfile_writer_init(gfile, gfs, "large", "text/html"); for(n=0; n < (LARGE / 1024); n++) { gridfile_write_buffer(gfile, buf + (n * 1024), 1024); } gridfile_writer_done( gfile ); test_gridfile(gfs, buf, LARGE, "large", "text/html"); gridfs_destroy(gfs); mongo_destroy(conn); }
void clsRasterData::outputToMongoDB(map<string,float> header, int nValid, float** position, float* value,string remoteFilename, gridfs* gfs) { float noData = -9999.0f; // prepare binary data int rows = int(header["NROWS"]); int cols = int(header["NCOLS"]); float *data = new float[rows*cols]; int index = 0; int dataIndex = 0; for (int i = 0; i < rows; ++i) { for (int j = 0; j < cols; ++j) { dataIndex = i*cols + j; if(index < nValid) { if(position[index][0] == i && position[index][1] == j) { data[dataIndex] = value[index]; index++; } else data[dataIndex] = noData; } else data[dataIndex] = noData; } } bson *p = (bson*)malloc(sizeof(bson)); bson_init(p); bson_append_string(p, "ID", remoteFilename.c_str()); bson_append_double(p, "CELLSIZE", header["CELLSIZE"]); bson_append_double(p, "NODATA_VALUE", noData); bson_append_double(p, "NCOLS", cols); bson_append_double(p, "NROWS", rows); bson_append_double(p, "XLLCENTER", header["XLLCENTER"]); bson_append_double(p, "YLLCENTER", header["YLLCENTER"]); bson_finish(p); gridfile gfile[1]; gridfile_writer_init(gfile, gfs, remoteFilename.c_str(), "float"); size_t iID = remoteFilename.find_first_of('_'); int subbasinID = atoi(remoteFilename.substr(0, iID).c_str()); gfile->id.ints[0] = subbasinID; for (int k = 0; k < rows; k++) { gridfile_write_buffer(gfile, (const char*)(data+cols*k), sizeof(float)*cols); } gridfile_set_metadata(gfile, p); int response = gridfile_writer_done(gfile); //cout << remoteFilename << "\t" << gfile->id.ints[0] << gfile->id.ints[1] << gfile->id.ints[2] << endl; gridfile_destroy(gfile); bson_destroy(p); free(p); delete data; }
static void test_large( void ) { mongo conn[1]; gridfs gfs[1]; gridfile gfile[1]; FILE *fd; size_t i, n; char *buffer = (char*)bson_malloc( LARGE ); char *read_buf = (char*)bson_malloc( LARGE ); gridfs_offset filesize = ( int64_t )1024 * ( int64_t )LARGE; mongo_write_concern wc; bson lastError; bson lastErrorCmd; srand( (unsigned int) time( NULL ) ); INIT_SOCKETS_FOR_WINDOWS; CONN_CLIENT_TEST; mongo_write_concern_init(&wc); wc.j = 1; mongo_write_concern_finish(&wc); mongo_set_write_concern(conn, &wc); GFS_INIT; fd = fopen( "bigfile", "r" ); if( fd ) { fclose( fd ); } else { /* Create a very large file */ fill_buffer_randomly( buffer, ( int64_t )LARGE ); fd = fopen( "bigfile", "w" ); for( i=0; i<1024; i++ ) { fwrite( buffer, 1, LARGE, fd ); } fclose( fd ); } /* Now read the file into GridFS */ gridfs_remove_filename( gfs, "bigfile" ); gridfs_store_file( gfs, "bigfile", "bigfile", "text/html", GRIDFILE_NOMD5 | GRIDFILE_COMPRESS); gridfs_find_filename( gfs, "bigfile", gfile ); ASSERT( strcmp( gridfile_get_filename( gfile ), "bigfile" ) == 0 ); ASSERT( gridfile_get_contentlength( gfile ) == filesize ); fd = fopen( "bigfile", "r" ); while( ( n = fread( buffer, 1, MEDIUM, fd ) ) != 0 ) { ASSERT( gridfile_read_buffer( gfile, read_buf, MEDIUM ) == n ); ASSERT( memcmp( buffer, read_buf, n ) == 0 ); } fclose( fd ); gridfile_destroy( gfile ); /* Read the file using the streaming interface */ gridfs_remove_filename( gfs, "bigfile" ); gridfs_remove_filename( gfs, "bigfile-stream" ); gridfile_writer_init( gfile, gfs, "bigfile-stream", "text/html", GRIDFILE_NOMD5 | GRIDFILE_COMPRESS ); mongo_write_concern_destroy( &wc ); mongo_write_concern_init(&wc); wc.j = 0; /* Let's reset write concern j field to zero, we will manually call getLastError with j = 1 */ mongo_write_concern_finish(&wc); mongo_set_write_concern(conn, &wc); fd = fopen( "bigfile", "r" ); i = 0; while( ( n = fread( buffer, 1, READ_WRITE_BUF_SIZE, fd ) ) != 0 ) { ASSERT( gridfile_write_buffer( gfile, buffer, n ) == n ); if(i++ % 10 == 0) { bson_init( &lastErrorCmd ); bson_append_int( &lastErrorCmd, "getLastError", 1); bson_append_int( &lastErrorCmd, "j", 1); bson_finish( &lastErrorCmd ); bson_init( &lastError ); mongo_run_command( conn, "test", &lastErrorCmd, &lastError ); bson_destroy( &lastError ); bson_destroy( &lastErrorCmd ); } } mongo_write_concern_destroy( &wc ); mongo_write_concern_init(&wc); wc.j = 1; /* Let's reset write concern j field to 1 */ mongo_write_concern_finish(&wc); mongo_set_write_concern(conn, &wc); fclose( fd ); gridfile_writer_done( gfile ); gridfs_find_filename( gfs, "bigfile-stream", gfile ); ASSERT( strcmp( gridfile_get_filename( gfile ), "bigfile-stream" ) == 0 ); ASSERT( gridfile_get_contentlength( gfile ) == filesize ); gridfs_remove_filename( gfs, "bigfile-stream" ); gridfs_destroy( gfs ); mongo_disconnect( conn ); mongo_destroy( conn ); bson_free( buffer ); bson_free( read_buf ); mongo_write_concern_destroy( &wc ); }
static void test_random_write2( void ) { mongo conn[1]; gridfs gfs[1]; gridfile gfile[1]; bson meta; char *buf = (char*)bson_malloc( LARGE ); char *zeroedbuf = (char*)bson_malloc( LARGE ); int n; if( buf == NULL ) { printf( "Failed to allocate" ); exit( 1 ); } srand( 123 ); // Init with a predictable value INIT_SOCKETS_FOR_WINDOWS; CONN_CLIENT_TEST; fill_buffer_randomly( buf, ( int64_t )LARGE ); memset( zeroedbuf, 0, LARGE ); bson_init_empty( &meta ); GFS_INIT; /* This portion of the test we will write zeroes by using new API gridfile_set_size function gridfile_expand tested implicitally by using gridfile_set_size making file larger */ gridfile_init( gfs, &meta, gfile ); gridfile_writer_init( gfile, gfs, "random_access", "text/html", 0 ); gridfile_set_size( gfile, LARGE ); // New API, this zero fills the file gridfile_writer_done( gfile ); test_gridfile( gfs, zeroedbuf, LARGE, "random_access", "text/html" ); // Test zero filled file /* This portion of the test we will write zeroes by using new API gridfile_set_size then we will truncate the file */ gridfile_init( gfs, &meta, gfile ); gridfile_writer_init( gfile, gfs, "random_access", "text/html", 0 ); gridfile_set_size( gfile, LARGE ); // New API, this zero fills the file with LARGE bytes gridfile_truncate( gfile, LARGE / 2 ); // Let's truncate the file now gridfile_writer_done( gfile ); test_gridfile( gfs, zeroedbuf, LARGE / 2, "random_access", "text/html" ); // Test zero filled file truncated by half /* Let's re-create the file, now let's randomly write real data */ gridfile_init( gfs, &meta, gfile ); gridfile_writer_init( gfile, gfs, "random_access", "text/html", 0 ); gridfile_set_size( gfile, LARGE ); // We need to reserve LARGE bytes on file before writing backwards /* Here we will write backwards the file with our random numbers. We will use a 3072 size buffer to cross over buffers given the fact 256K is not a multiple of 3072. Let's stress the buffering logic a little bit */ for( n = LARGE / 3072 - 1; n >= 0; n-- ) { gridfile_seek( gfile, 3072 * n ); ASSERT( gridfile_write_buffer( gfile, buf + ( n * 3072 ), 3072 ) == 3072 ); } gridfile_writer_done( gfile ); test_gridfile( gfs, buf, LARGE, "random_access", "text/html" ); gridfs_destroy( gfs ); mongo_destroy( conn ); free( buf ); free( zeroedbuf ); }
static void test_random_write(void) { mongo conn[1]; gridfs gfs[1]; gridfile* gfile; char *data_before = (char*)bson_malloc( UPPER ); char *random_data = (char*)bson_malloc( UPPER ); char *buf = (char*) bson_malloc( UPPER ); int64_t i; FILE *fd; srand((unsigned int) time( NULL ) ); INIT_SOCKETS_FOR_WINDOWS; CONN_CLIENT_TEST; GFS_INIT; fill_buffer_randomly( data_before, UPPER ); fill_buffer_randomly( random_data, UPPER ); for ( i = LOWER; i <= UPPER; i += DELTA ) { int64_t j = i / 2 - 3; gridfs_offset bytes_to_write_first; int n; /* Input from buffer */ gridfs_store_buffer( gfs, data_before, i, "input-buffer", "text/html", GRIDFILE_DEFAULT ); if ( i > DEFAULT_CHUNK_SIZE * 4 ) { n = DEFAULT_CHUNK_SIZE * 3 + 6; memcpy(&data_before[j], random_data, n); // Let's overwrite the buffer with bytes crossing multiple chunks bytes_to_write_first = 10; } else { n = 6; memcpy(random_data, "123456", n); strncpy(&data_before[j], random_data, n); // Let's overwrite the buffer with a few some bytes bytes_to_write_first = 0; } gfile = gridfile_create(); ASSERT(gridfs_find_filename(gfs, "input-buffer", gfile) == 0); gridfile_writer_init(gfile, gfs, "input-buffer", "text/html", GRIDFILE_DEFAULT ); gridfile_seek(gfile, j); // Seek into the same buffer position within the GridFS file if ( bytes_to_write_first ) { ASSERT( gridfile_write_buffer(gfile, random_data, bytes_to_write_first) == bytes_to_write_first ); // Let's write 10 bytes first, and later the rest } ASSERT( gridfile_write_buffer(gfile, &random_data[bytes_to_write_first], n - bytes_to_write_first) == n - bytes_to_write_first ); // Try to write to the existing GridFS file on the position given by j gridfile_seek(gfile, j); gridfile_read_buffer( gfile, buf, n ); ASSERT(memcmp( buf, &data_before[j], n) == 0); gridfile_writer_done(gfile); ASSERT(gfile->pos == (gridfs_offset)(j + n)); gridfile_dealloc(gfile); test_gridfile( gfs, data_before, j + n > i ? j + n : i, "input-buffer", "text/html" ); /* Input from file */ fd = fopen( "input-file", "w" ); fwrite( data_before, sizeof( char ), (size_t) (j + n > i ? j + n : i), fd ); fclose( fd ); gridfs_store_file( gfs, "input-file", "input-file", "text/html", GRIDFILE_DEFAULT ); test_gridfile( gfs, data_before, j + n > i ? j + n : i, "input-file", "text/html" ); } gridfs_destroy( gfs ); mongo_disconnect( conn ); mongo_destroy( conn ); free( data_before ); free( random_data ); free( buf ); /* Clean up files. */ gridfs_test_unlink( "input-file" ); gridfs_test_unlink( "output" ); }
SEXP mongo_gridfile_writer_write(SEXP gfw, SEXP raw) { gridfile* gfile = _checkGridfileWriter(gfw); int len = LENGTH(raw); if (len) gridfile_write_buffer(gfile, (char*)RAW(raw), len); return R_NilValue; }