SEXP mongo_gridfile_seek(SEXP gfile, SEXP offset) { gridfile* _gfile = _checkGridfile(gfile); gridfs_offset _offset = asReal(offset); SEXP ret; PROTECT(ret = allocVector(REALSXP, 1)); REAL(ret)[0] = gridfile_seek(_gfile, _offset); return ret; }
static void test_gridfile( gridfs *gfs, char *data_before, int64_t length, char *filename, char *content_type ) { gridfile gfile[1]; FILE *stream; #ifdef DYING mongo_md5_state_t pms[1]; mongo_md5_byte_t digest[16]; #endif char hex_digest[33]; int64_t i = length; int n; char *data_after = (char*)bson_malloc( LARGE ); int truncBytes; char* lowerName; ASSERT(gridfs_find_filename( gfs, filename, gfile ) == MONGO_OK); ASSERT( gridfile_exists( gfile ) ); stream = fopen( "output", "w+" ); gridfile_write_file( gfile, stream ); fseek( stream, 0, SEEK_SET ); ASSERT( fread( data_after, (size_t)length, sizeof( char ), stream ) ); fclose( stream ); ASSERT( memcmp( data_before, data_after, (size_t)length ) == 0 ); gridfile_read_buffer( gfile, data_after, length ); ASSERT( memcmp( data_before, data_after, (size_t)length ) == 0 ); lowerName = (char*) bson_malloc( (int)strlen( filename ) + 1); strcpy( lowerName, filename ); _strlwr( lowerName ); ASSERT( strcmp( gridfile_get_filename( gfile ), lowerName ) == 0 ); bson_free( lowerName ); ASSERT( gridfile_get_contentlength( gfile ) == (size_t)length ); ASSERT( gridfile_get_chunksize( gfile ) == DEFAULT_CHUNK_SIZE ); ASSERT( strcmp( gridfile_get_contenttype( gfile ), content_type ) == 0 ) ; ASSERT( memcmp( data_before, data_after, (size_t)length ) == 0 ); if( !( gfile->flags & GRIDFILE_COMPRESS ) ) { #ifdef DYING mongo_md5_init( pms ); n = 0; while( i > INT_MAX ) { mongo_md5_append( pms, ( const mongo_md5_byte_t * )data_before + ( n * INT_MAX ), INT_MAX ); i -= INT_MAX; n += 1; } if( i > 0 ) mongo_md5_append( pms, ( const mongo_md5_byte_t * )data_before + ( n * INT_MAX ), (int)i ); mongo_md5_finish( pms, digest ); digest2hex( digest, hex_digest ); #else { DIGEST_CTX ctx = rpmDigestInit(PGPHASHALGO_MD5, RPMDIGEST_NONE); const char * _digest = NULL; int xx; while( i > INT_MAX ) { xx = rpmDigestUpdate(ctx, (char *)data_before + (n*INT_MAX), INT_MAX); i -= INT_MAX; n += 1; } xx = rpmDigestFinal(ctx, &_digest, NULL, 1); strncpy(hex_digest, _digest, 32+1); hex_digest[32] = '\0'; _digest = _free(_digest); } #endif ASSERT( strcmp( gridfile_get_md5( gfile ), hex_digest ) == 0 ); } truncBytes = (int) (length > DEFAULT_CHUNK_SIZE * 4 ? length - DEFAULT_CHUNK_SIZE * 2 - 13 : 23); gridfile_writer_init( gfile, gfs, filename, content_type, GRIDFILE_DEFAULT); ASSERT( gridfile_truncate(gfile, (size_t)(length - truncBytes)) == (size_t)(length - truncBytes)); gridfile_writer_done( gfile ); gridfile_seek(gfile, 0); ASSERT( gridfile_get_contentlength( gfile ) == (size_t)(length - truncBytes) ); ASSERT( gridfile_read_buffer( gfile, data_after, length ) == (size_t)(length - truncBytes)); ASSERT( memcmp( data_before, data_after, (size_t)(length - truncBytes) ) == 0 ); gridfile_writer_init( gfile, gfs, filename, content_type, GRIDFILE_DEFAULT); gridfile_truncate(gfile, 0); gridfile_writer_done( gfile ); ASSERT( gridfile_get_contentlength( gfile ) == 0 ); ASSERT( gridfile_read_buffer( gfile, data_after, length ) == 0 ); gridfile_destroy( gfile ); ASSERT( gridfs_remove_filename( gfs, filename ) == MONGO_OK ); free( data_after ); gridfs_test_unlink( "output" ); }
static void test_random_write2( void ) { mongo conn[1]; gridfs gfs[1]; gridfile gfile[1]; bson meta; char *buf = (char*)bson_malloc( LARGE ); char *zeroedbuf = (char*)bson_malloc( LARGE ); int n; if( buf == NULL ) { printf( "Failed to allocate" ); exit( 1 ); } srand( 123 ); // Init with a predictable value INIT_SOCKETS_FOR_WINDOWS; CONN_CLIENT_TEST; fill_buffer_randomly( buf, ( int64_t )LARGE ); memset( zeroedbuf, 0, LARGE ); bson_init_empty( &meta ); GFS_INIT; /* This portion of the test we will write zeroes by using new API gridfile_set_size function gridfile_expand tested implicitally by using gridfile_set_size making file larger */ gridfile_init( gfs, &meta, gfile ); gridfile_writer_init( gfile, gfs, "random_access", "text/html", 0 ); gridfile_set_size( gfile, LARGE ); // New API, this zero fills the file gridfile_writer_done( gfile ); test_gridfile( gfs, zeroedbuf, LARGE, "random_access", "text/html" ); // Test zero filled file /* This portion of the test we will write zeroes by using new API gridfile_set_size then we will truncate the file */ gridfile_init( gfs, &meta, gfile ); gridfile_writer_init( gfile, gfs, "random_access", "text/html", 0 ); gridfile_set_size( gfile, LARGE ); // New API, this zero fills the file with LARGE bytes gridfile_truncate( gfile, LARGE / 2 ); // Let's truncate the file now gridfile_writer_done( gfile ); test_gridfile( gfs, zeroedbuf, LARGE / 2, "random_access", "text/html" ); // Test zero filled file truncated by half /* Let's re-create the file, now let's randomly write real data */ gridfile_init( gfs, &meta, gfile ); gridfile_writer_init( gfile, gfs, "random_access", "text/html", 0 ); gridfile_set_size( gfile, LARGE ); // We need to reserve LARGE bytes on file before writing backwards /* Here we will write backwards the file with our random numbers. We will use a 3072 size buffer to cross over buffers given the fact 256K is not a multiple of 3072. Let's stress the buffering logic a little bit */ for( n = LARGE / 3072 - 1; n >= 0; n-- ) { gridfile_seek( gfile, 3072 * n ); ASSERT( gridfile_write_buffer( gfile, buf + ( n * 3072 ), 3072 ) == 3072 ); } gridfile_writer_done( gfile ); test_gridfile( gfs, buf, LARGE, "random_access", "text/html" ); gridfs_destroy( gfs ); mongo_destroy( conn ); free( buf ); free( zeroedbuf ); }
static void test_random_write(void) { mongo conn[1]; gridfs gfs[1]; gridfile* gfile; char *data_before = (char*)bson_malloc( UPPER ); char *random_data = (char*)bson_malloc( UPPER ); char *buf = (char*) bson_malloc( UPPER ); int64_t i; FILE *fd; srand((unsigned int) time( NULL ) ); INIT_SOCKETS_FOR_WINDOWS; CONN_CLIENT_TEST; GFS_INIT; fill_buffer_randomly( data_before, UPPER ); fill_buffer_randomly( random_data, UPPER ); for ( i = LOWER; i <= UPPER; i += DELTA ) { int64_t j = i / 2 - 3; gridfs_offset bytes_to_write_first; int n; /* Input from buffer */ gridfs_store_buffer( gfs, data_before, i, "input-buffer", "text/html", GRIDFILE_DEFAULT ); if ( i > DEFAULT_CHUNK_SIZE * 4 ) { n = DEFAULT_CHUNK_SIZE * 3 + 6; memcpy(&data_before[j], random_data, n); // Let's overwrite the buffer with bytes crossing multiple chunks bytes_to_write_first = 10; } else { n = 6; memcpy(random_data, "123456", n); strncpy(&data_before[j], random_data, n); // Let's overwrite the buffer with a few some bytes bytes_to_write_first = 0; } gfile = gridfile_create(); ASSERT(gridfs_find_filename(gfs, "input-buffer", gfile) == 0); gridfile_writer_init(gfile, gfs, "input-buffer", "text/html", GRIDFILE_DEFAULT ); gridfile_seek(gfile, j); // Seek into the same buffer position within the GridFS file if ( bytes_to_write_first ) { ASSERT( gridfile_write_buffer(gfile, random_data, bytes_to_write_first) == bytes_to_write_first ); // Let's write 10 bytes first, and later the rest } ASSERT( gridfile_write_buffer(gfile, &random_data[bytes_to_write_first], n - bytes_to_write_first) == n - bytes_to_write_first ); // Try to write to the existing GridFS file on the position given by j gridfile_seek(gfile, j); gridfile_read_buffer( gfile, buf, n ); ASSERT(memcmp( buf, &data_before[j], n) == 0); gridfile_writer_done(gfile); ASSERT(gfile->pos == (gridfs_offset)(j + n)); gridfile_dealloc(gfile); test_gridfile( gfs, data_before, j + n > i ? j + n : i, "input-buffer", "text/html" ); /* Input from file */ fd = fopen( "input-file", "w" ); fwrite( data_before, sizeof( char ), (size_t) (j + n > i ? j + n : i), fd ); fclose( fd ); gridfs_store_file( gfs, "input-file", "input-file", "text/html", GRIDFILE_DEFAULT ); test_gridfile( gfs, data_before, j + n > i ? j + n : i, "input-file", "text/html" ); } gridfs_destroy( gfs ); mongo_disconnect( conn ); mongo_destroy( conn ); free( data_before ); free( random_data ); free( buf ); /* Clean up files. */ gridfs_test_unlink( "input-file" ); gridfs_test_unlink( "output" ); }
EXPORT double mongo_gridfile_seek(struct gridfile_* gf, double offset) { return (double)gridfile_seek((gridfile*)gf, (gridfs_offset)offset); }
void test_gridfile( gridfs *gfs, char *data_before, int64_t length, char *filename, char *content_type ) { gridfile gfile[1]; FILE *stream; mongo_md5_state_t pms[1]; mongo_md5_byte_t digest[16]; char hex_digest[33]; int64_t i = length; int n; char *data_after = (char*)bson_malloc( LARGE ); int truncBytes; char* lowerName; ASSERT(gridfs_find_filename( gfs, filename, gfile ) == MONGO_OK); ASSERT( gridfile_exists( gfile ) ); stream = fopen( "output", "w+" ); gridfile_write_file( gfile, stream ); fseek( stream, 0, SEEK_SET ); ASSERT( fread( data_after, (size_t)length, sizeof( char ), stream ) ); fclose( stream ); ASSERT( memcmp( data_before, data_after, (size_t)length ) == 0 ); gridfile_read( gfile, length, data_after ); ASSERT( memcmp( data_before, data_after, (size_t)length ) == 0 ); lowerName = (char*) bson_malloc( (int)strlen( filename ) + 1); strcpy( lowerName, filename); _strlwr( lowerName ); ASSERT( strcmp( gridfile_get_filename( gfile ), lowerName ) == 0 ); bson_free( lowerName ); ASSERT( gridfile_get_contentlength( gfile ) == (size_t)length ); ASSERT( gridfile_get_chunksize( gfile ) == DEFAULT_CHUNK_SIZE ); ASSERT( strcmp( gridfile_get_contenttype( gfile ), content_type ) == 0 ) ; ASSERT( memcmp( data_before, data_after, (size_t)length ) == 0 ); if( !( gfile->flags & GRIDFILE_COMPRESS ) ) { mongo_md5_init( pms ); n = 0; while( i > INT_MAX ) { mongo_md5_append( pms, ( const mongo_md5_byte_t * )data_before + ( n * INT_MAX ), INT_MAX ); i -= INT_MAX; n += 1; } if( i > 0 ) mongo_md5_append( pms, ( const mongo_md5_byte_t * )data_before + ( n * INT_MAX ), (int)i ); mongo_md5_finish( pms, digest ); digest2hex( digest, hex_digest ); ASSERT( strcmp( gridfile_get_md5( gfile ), hex_digest ) == 0 ); } truncBytes = (int) (length > DEFAULT_CHUNK_SIZE * 4 ? length - DEFAULT_CHUNK_SIZE * 2 - 13 : 23); gridfile_writer_init( gfile, gfs, filename, content_type, GRIDFILE_DEFAULT); ASSERT( gridfile_truncate(gfile, (size_t)(length - truncBytes)) == (size_t)(length - truncBytes)); gridfile_writer_done( gfile ); gridfile_seek(gfile, 0); ASSERT( gridfile_get_contentlength( gfile ) == (size_t)(length - truncBytes) ); ASSERT( gridfile_read( gfile, length, data_after ) == (size_t)(length - truncBytes)); ASSERT( memcmp( data_before, data_after, (size_t)(length - truncBytes) ) == 0 ); gridfile_writer_init( gfile, gfs, filename, content_type, GRIDFILE_DEFAULT); gridfile_truncate(gfile, 0); gridfile_writer_done( gfile ); ASSERT( gridfile_get_contentlength( gfile ) == 0 ); ASSERT( gridfile_read( gfile, length, data_after ) == 0 ); gridfile_destroy( gfile ); gridfs_remove_filename( gfs, filename ); free( data_after ); _unlink( "output" ); }