void test_streaming( void ) {
mongo conn[1];
gridfs gfs[1];
gridfile gfile[1];
char *medium = (char*)bson_malloc( 2*MEDIUM );
char *small = (char*)bson_malloc( LOWER );
char *buf = (char*)bson_malloc( LARGE );
int n;
if( buf == NULL || small == NULL ) {
printf( "Failed to allocate" );
exit( 1 );
}
srand( (unsigned int)time( NULL ) );
INIT_SOCKETS_FOR_WINDOWS;
if ( mongo_client( conn , TEST_SERVER, 27017 ) ) {
printf( "failed to connect 3\n" );
exit( 1 );
}
fill_buffer_randomly( medium, ( int64_t )2 * MEDIUM );
fill_buffer_randomly( small, ( int64_t )LOWER );
fill_buffer_randomly( buf, ( int64_t )LARGE );
gridfs_init( conn, "test", "fs", gfs );
gridfile_writer_init( gfile, gfs, "medium", "text/html", GRIDFILE_DEFAULT );
gridfile_write_buffer( gfile, medium, MEDIUM );
gridfile_write_buffer( gfile, medium + MEDIUM, MEDIUM );
gridfile_writer_done( gfile );
test_gridfile( gfs, medium, 2 * MEDIUM, "medium", "text/html" );
gridfs_destroy( gfs );
gridfs_init( conn, "test", "fs", gfs );
gridfs_store_buffer( gfs, small, LOWER, "small", "text/html", GRIDFILE_DEFAULT );
test_gridfile( gfs, small, LOWER, "small", "text/html" );
gridfs_destroy( gfs );
gridfs_init( conn, "test", "fs", gfs );
gridfs_remove_filename( gfs, "large" );
gridfile_writer_init( gfile, gfs, "large", "text/html", GRIDFILE_DEFAULT );
for( n=0; n < ( LARGE / 1024 ); n++ ) {
gridfile_write_buffer( gfile, buf + ( n * 1024 ), 1024 );
}
gridfile_writer_done( gfile );
test_gridfile( gfs, buf, LARGE, "large", "text/html" );
gridfs_destroy( gfs );
mongo_destroy( conn );
free( buf );
free( small );
free( medium );
}
static void
upload_init (upload_test_t *upload_test)
{
gridfs_init (upload_test, "TestGridFsUpload");
upload_test->base.before = upload_before;
upload_test->base.task = upload_task;
}
static void
download_init (download_test_t *download_test)
{
gridfs_init (&download_test->base, "TestGridFsDownload");
download_test->base.base.setup = download_setup;
download_test->base.base.task = download_task;
}
void test_large() {
mongo_connection conn[1];
mongo_connection_options opts;
gridfs gfs[1];
gridfile gfile[1];
FILE *fd;
int i, n;
char buffer[LARGE];
int64_t filesize = (int64_t)1024 * (int64_t)LARGE;
srand(time(NULL));
INIT_SOCKETS_FOR_WINDOWS;
strncpy(opts.host, "127.0.0.1", 255);
opts.host[254] = '\0';
opts.port = 27017;
if (mongo_connect( conn , &opts )) {
printf("failed to connect\n");
exit(1);
}
gridfs_init(conn, "test", "fs", gfs);
/* Create a very large file */
fill_buffer_randomly(buffer, (int64_t)LARGE);
fd = fopen("bigfile", "w");
for(i=0; i<1024; i++) {
fwrite(buffer, 1, LARGE, fd);
}
fclose(fd);
/* Now read the file into GridFS */
gridfs_store_file(gfs, "bigfile", "bigfile", "text/html");
gridfs_find_filename(gfs, "bigfile", gfile);
ASSERT( strcmp( gridfile_get_filename( gfile ), "bigfile" ) == 0 );
ASSERT( gridfile_get_contentlength( gfile ) == filesize );
/* Read the file using the streaming interface */
gridfile_writer_init( gfile, gfs, "bigfile-stream", "text/html");
fd = fopen("bigfile", "r");
while((n = fread(buffer, 1, 1024, fd)) != 0) {
gridfile_write_buffer(gfile, buffer, n);
}
gridfile_writer_done( gfile );
gridfs_find_filename(gfs, "bigfile-stream", gfile);
ASSERT( strcmp( gridfile_get_filename( gfile ), "bigfile-stream" ) == 0 );
ASSERT( gridfile_get_contentlength( gfile ) == filesize );
gridfs_destroy(gfs);
mongo_destroy(conn);
}
void test_large() {
mongo_connection conn[1];
gridfs gfs[1];
gridfile gfile[1];
FILE *fd;
int i, n;
char *buffer = malloc( LARGE );
if( buffer == NULL ) {
printf("Failed to allocate memory.");
exit(1);
}
uint64_t filesize = (uint64_t)1024 * (uint64_t)LARGE;
srand(time(NULL));
INIT_SOCKETS_FOR_WINDOWS;
if (mongo_connect( conn, TEST_SERVER, 27017 )){
printf("failed to connect 1\n");
exit(1);
}
gridfs_init(conn, "test", "fs", gfs);
/* Create a very large file */
fill_buffer_randomly(buffer, (uint64_t)LARGE);
fd = fopen("bigfile", "w");
for(i=0; i<1024; i++) {
fwrite(buffer, 1, LARGE, fd);
}
fclose(fd);
/* Now read the file into GridFS */
gridfs_store_file(gfs, "bigfile", "bigfile", "text/html");
gridfs_find_filename(gfs, "bigfile", gfile);
ASSERT( strcmp( gridfile_get_filename( gfile ), "bigfile" ) == 0 );
ASSERT( gridfile_get_contentlength( gfile ) == filesize );
/* Read the file using the streaming interface */
gridfile_writer_init( gfile, gfs, "bigfile-stream", "text/html");
fd = fopen("bigfile", "r");
while((n = fread(buffer, 1, 1024, fd)) != 0) {
gridfile_write_buffer(gfile, buffer, n);
}
gridfile_writer_done( gfile );
gridfs_find_filename(gfs, "bigfile-stream", gfile);
ASSERT( strcmp( gridfile_get_filename( gfile ), "bigfile-stream" ) == 0 );
ASSERT( gridfile_get_contentlength( gfile ) == filesize );
gridfs_destroy(gfs);
mongo_disconnect(conn);
mongo_destroy(conn);
}
EXPORT int mongo_gridfs_create(struct mongo_* conn, char* db, char* prefix, struct gridfs_** gfs) {
gridfs* gfs_ = (gridfs*)malloc(sizeof(gridfs));
if (gridfs_init((mongo*)conn, db, prefix, gfs_) != MONGO_OK) {
free(gfs_);
return 0;
}
*gfs = (struct gridfs_*)gfs_;
return 1;
}
void test_streaming() {
mongo_connection conn[1];
gridfs gfs[1];
gridfile gfile[1];
char *buf = malloc( LARGE );
char *small = malloc( LOWER );
if( buf == NULL || small == NULL ) {
printf("Failed to allocate");
exit(1);
}
int n;
srand(time(NULL));
INIT_SOCKETS_FOR_WINDOWS;
if (mongo_connect( conn , TEST_SERVER, 27017 )){
printf("failed to connect 3\n");
exit(1);
}
fill_buffer_randomly(small, (uint64_t)LOWER);
fill_buffer_randomly(buf, (uint64_t)LARGE);
gridfs_init(conn, "test", "fs", gfs);
gridfs_store_buffer(gfs, small, LOWER, "small", "text/html");
test_gridfile(gfs, small, LOWER, "small", "text/html");
gridfs_destroy(gfs);
gridfs_init(conn, "test", "fs", gfs);
gridfile_writer_init(gfile, gfs, "large", "text/html");
for(n=0; n < (LARGE / 1024); n++) {
gridfile_write_buffer(gfile, buf + (n * 1024), 1024);
}
gridfile_writer_done( gfile );
test_gridfile(gfs, buf, LARGE, "large", "text/html");
gridfs_destroy(gfs);
mongo_destroy(conn);
free(buf);
free(small);
}
void test_streaming() {
mongo_connection conn[1];
mongo_connection_options opts;
gridfs gfs[1];
gridfile gfile[1];
char buf[LARGE];
char small[LOWER];
int n;
srand(time(NULL));
INIT_SOCKETS_FOR_WINDOWS;
strncpy(opts.host, "127.0.0.1", 255);
opts.host[254] = '\0';
opts.port = 27017;
if (mongo_connect( conn , &opts )) {
printf("failed to connect\n");
exit(1);
}
fill_buffer_randomly(small, (int64_t)LOWER);
fill_buffer_randomly(buf, (int64_t)LARGE);
gridfs_init(conn, "test", "fs", gfs);
gridfs_store_buffer(gfs, small, LOWER, "small", "text/html");
test_gridfile(gfs, small, LOWER, "small", "text/html");
gridfs_destroy(gfs);
gridfs_init(conn, "test", "fs", gfs);
gridfile_writer_init(gfile, gfs, "large", "text/html");
for(n=0; n < (LARGE / 1024); n++) {
gridfile_write_buffer(gfile, buf + (n * 1024), 1024);
}
gridfile_writer_done( gfile );
test_gridfile(gfs, buf, LARGE, "large", "text/html");
gridfs_destroy(gfs);
mongo_destroy(conn);
}
void test_basic( void ) {
mongo conn[1];
gridfs gfs[1];
char *data_before = (char*)bson_malloc( UPPER );
int64_t i;
FILE *fd;
srand((unsigned int) time( NULL ) );
INIT_SOCKETS_FOR_WINDOWS;
if ( mongo_client( conn, TEST_SERVER, 27017 ) ) {
printf( "failed to connect 2\n" );
exit( 1 );
}
gridfs_init( conn, "test", "fs", gfs );
fill_buffer_randomly( data_before, UPPER );
for ( i = LOWER; i <= UPPER; i += DELTA ) {
/* Input from buffer */
gridfs_store_buffer( gfs, data_before, i, "input-buffer", "text/html", GRIDFILE_COMPRESS );
test_gridfile( gfs, data_before, i, "input-buffer", "text/html" );
/* Input from file */
fd = fopen( "input-file", "w" );
fwrite( data_before, sizeof( char ), (size_t)i, fd );
fclose( fd );
gridfs_store_file( gfs, "input-file", "input-file", "text/html", GRIDFILE_DEFAULT );
test_gridfile( gfs, data_before, i, "input-file", "text/html" );
gfs->caseInsensitive = 1;
gridfs_store_file( gfs, "input-file", "input-file", "text/html", GRIDFILE_DEFAULT );
test_gridfile( gfs, data_before, i, "inPut-file", "text/html" );
}
gridfs_destroy( gfs );
mongo_disconnect( conn );
mongo_destroy( conn );
free( data_before );
/* Clean up files. */
_unlink( "input-file" );
_unlink( "output" );
}
int main(void) {
mongo_connection conn[1];
mongo_connection_options opts;
gridfs gfs[1];
char data_before[UPPER];
size_t i;
FILE *fd;
srand(time(NULL));
INIT_SOCKETS_FOR_WINDOWS;
/* strncpy(opts.host, TEST_SERVER, 255);*/
strncpy(opts.host, "127.0.0.1", 255);
opts.host[254] = '\0';
opts.port = 27017;
if (mongo_connect( conn , &opts )){
printf("failed to connect\n");
exit(1);
}
gridfs_init(conn, "test", "fs", gfs);
for (i = LOWER; i <= UPPER; i+=DELTA) {
fill_buffer_randomly(data_before, i);
/* Input from buffer */
gridfs_store_buffer(gfs, data_before, i, "input-buffer", "text/html");
test_gridfile(gfs, data_before, i, "input-buffer", "text/html");
/* Input from file */
fd = fopen("input-file", "w");
fwrite(data_before, sizeof(char), i, fd);
fclose(fd);
gridfs_store_file(gfs, "input-file", "input-file", "text/html");
test_gridfile(gfs, data_before, i, "input-file", "text/html");
}
gridfs_destroy(gfs);
mongo_cmd_drop_db(conn, "test");
mongo_destroy(conn);
return 0;
}
void test_basic() {
mongo_connection conn[1];
gridfs gfs[1];
char *data_before = malloc( UPPER );
if( data_before == NULL ) {
printf("Failed to allocate");
exit(1);
}
uint64_t i;
FILE *fd;
srand(time(NULL));
INIT_SOCKETS_FOR_WINDOWS;
if (mongo_connect( conn, TEST_SERVER, 27017 )){
printf("failed to connect 2\n");
exit(1);
}
gridfs_init(conn, "test", "fs", gfs);
fill_buffer_randomly( data_before, UPPER );
for (i = LOWER; i <= UPPER; i += DELTA) {
/* Input from buffer */
gridfs_store_buffer(gfs, data_before, i, "input-buffer", "text/html");
test_gridfile(gfs, data_before, i, "input-buffer", "text/html");
/* Input from file */
fd = fopen("input-file", "w");
fwrite(data_before, sizeof(char), i, fd);
fclose(fd);
gridfs_store_file(gfs, "input-file", "input-file", "text/html");
test_gridfile(gfs, data_before, i, "input-file", "text/html");
}
gridfs_destroy(gfs);
mongo_disconnect(conn);
mongo_destroy(conn);
free( data_before );
}
SEXP mongo_gridfs_create(SEXP mongo_conn, SEXP db, SEXP prefix) {
mongo* conn = _checkMongo(mongo_conn);
const char* _db = CHAR(STRING_ELT(db, 0));
const char* _prefix = CHAR(STRING_ELT(prefix, 0));
gridfs* gfs = Calloc(1, gridfs);
if (gridfs_init(conn, _db, _prefix, gfs) != MONGO_OK) {
Free(gfs);
return R_NilValue;
}
SEXP ret, ptr, cls;
PROTECT(ret = allocVector(INTSXP, 1));
INTEGER(ret)[0] = 0;
/* prevent GC on connection object while gfs alive */
setAttrib(ret, sym_mongo, mongo_conn);
ptr = R_MakeExternalPtr(gfs, sym_mongo_gridfs, R_NilValue);
PROTECT(ptr);
R_RegisterCFinalizerEx(ptr, mongoGridfsFinalizer, TRUE);
setAttrib(ret, sym_mongo_gridfs, ptr);
PROTECT(cls = allocVector(STRSXP, 1));
SET_STRING_ELT(cls, 0, mkChar("mongo.gridfs"));
classgets(ret, cls);
UNPROTECT(3);
return ret;
}
static ngx_int_t ngx_http_gridfs_handler(ngx_http_request_t* request) {
ngx_http_gridfs_loc_conf_t* gridfs_conf;
ngx_http_core_loc_conf_t* core_conf;
ngx_buf_t* buffer;
ngx_chain_t out;
ngx_str_t location_name;
ngx_str_t full_uri;
char* value;
ngx_http_mongo_connection_t *mongo_conn;
gridfs gfs;
gridfile gfile;
gridfs_offset length;
ngx_uint_t numchunks;
char* contenttype;
char* md5;
bson_date_t last_modified;
volatile ngx_uint_t i;
ngx_int_t rc = NGX_OK;
bson query;
bson_oid_t oid;
mongo_cursor ** cursors;
gridfs_offset chunk_len;
const char * chunk_data;
bson_iterator it;
bson chunk;
ngx_pool_cleanup_t* gridfs_cln;
ngx_http_gridfs_cleanup_t* gridfs_clndata;
int status;
volatile ngx_uint_t e = FALSE;
volatile ngx_uint_t ecounter = 0;
uint64_t range_start = 0;
uint64_t range_end = 0;
uint64_t current_buf_pos = 0;
gridfs_conf = ngx_http_get_module_loc_conf(request, ngx_http_gridfs_module);
core_conf = ngx_http_get_module_loc_conf(request, ngx_http_core_module);
// ---------- ENSURE MONGO CONNECTION ---------- //
mongo_conn = ngx_http_get_mongo_connection( gridfs_conf->mongo );
if (mongo_conn == NULL) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Mongo Connection not found: \"%V\"", &gridfs_conf->mongo);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
if (mongo_conn->conn.connected == 0) {
if (ngx_http_mongo_reconnect(request->connection->log, mongo_conn) == NGX_ERROR) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Could not connect to mongo: \"%V\"", &gridfs_conf->mongo);
if(mongo_conn->conn.connected) { mongo_disconnect(&mongo_conn->conn); }
return NGX_HTTP_SERVICE_UNAVAILABLE;
}
if (ngx_http_mongo_reauth(request->connection->log, mongo_conn) == NGX_ERROR) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Failed to reauth to mongo: \"%V\"", &gridfs_conf->mongo);
if(mongo_conn->conn.connected) { mongo_disconnect(&mongo_conn->conn); }
return NGX_HTTP_SERVICE_UNAVAILABLE;
}
}
// ---------- RETRIEVE KEY ---------- //
location_name = core_conf->name;
full_uri = request->uri;
if (full_uri.len < location_name.len) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Invalid location name or uri.");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
value = (char*)malloc(sizeof(char) * (full_uri.len - location_name.len + 1));
if (value == NULL) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Failed to allocate memory for value buffer.");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
memcpy(value, full_uri.data + location_name.len, full_uri.len - location_name.len);
value[full_uri.len - location_name.len] = '\0';
if (!url_decode(value)) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Malformed request.");
free(value);
return NGX_HTTP_BAD_REQUEST;
}
// ---------- RETRIEVE GRIDFILE ---------- //
bson_init(&query);
switch (gridfs_conf->type) {
case BSON_OID:
bson_oid_from_string(&oid, value);
bson_append_oid(&query, (char*)gridfs_conf->field.data, &oid);
break;
case BSON_INT:
bson_append_int(&query, (char*)gridfs_conf->field.data, ngx_atoi((u_char*)value, strlen(value)));
break;
case BSON_STRING:
bson_append_string(&query, (char*)gridfs_conf->field.data, value);
break;
}
bson_finish(&query);
do {
e = FALSE;
if (gridfs_init(&mongo_conn->conn,
(const char*)gridfs_conf->db.data,
(const char*)gridfs_conf->root_collection.data,
&gfs) != MONGO_OK
|| (status = gridfs_find_query(&gfs, &query, &gfile) == MONGO_ERROR)) {
e = TRUE; ecounter++;
if (ecounter > MONGO_MAX_RETRIES_PER_REQUEST
|| ngx_http_mongo_reconnect(request->connection->log, mongo_conn) == NGX_ERROR
|| ngx_http_mongo_reauth(request->connection->log, mongo_conn) == NGX_ERROR) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Mongo connection dropped, could not reconnect");
if(&mongo_conn->conn.connected) { mongo_disconnect(&mongo_conn->conn); }
bson_destroy(&query);
free(value);
return NGX_HTTP_SERVICE_UNAVAILABLE;
}
}
} while (e);
bson_destroy(&query);
free(value);
/* Get information about the file */
length = gridfile_get_contentlength(&gfile);
numchunks = gridfile_get_numchunks(&gfile);
// NaN workaround
if (numchunks > INT_MAX)
{
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return NGX_HTTP_NOT_FOUND;
}
contenttype = (char*)gridfile_get_contenttype(&gfile);
md5 = (char*)gridfile_get_md5(&gfile);
last_modified = gridfile_get_uploaddate(&gfile);
// ---------- Partial Range
// set follow-fork-mode child
// attach (pid)
// break ngx_http_gridfs_module.c:959
if (request->headers_in.range) {
gridfs_parse_range(request, &request->headers_in.range->value, &range_start, &range_end, length);
}
// ---------- SEND THE HEADERS ---------- //
if (range_start == 0 && range_end == 0) {
request->headers_out.status = NGX_HTTP_OK;
request->headers_out.content_length_n = length;
} else {
request->headers_out.status = NGX_HTTP_PARTIAL_CONTENT;
request->headers_out.content_length_n = length;
//request->headers_out.content_range = range_end - range_start + 1;
ngx_table_elt_t *content_range;
content_range = ngx_list_push(&request->headers_out.headers);
if (content_range == NULL) {
return NGX_ERROR;
}
request->headers_out.content_range = content_range;
content_range->hash = 1;
ngx_str_set(&content_range->key, "Content-Range");
content_range->value.data = ngx_pnalloc(request->pool,sizeof("bytes -/") - 1 + 3 * NGX_OFF_T_LEN);
if (content_range->value.data == NULL) {
return NGX_ERROR;
}
/* "Content-Range: bytes SSSS-EEEE/TTTT" header */
content_range->value.len = ngx_sprintf(content_range->value.data,
"bytes %O-%O/%O",
range_start, range_end,
request->headers_out.content_length_n)
- content_range->value.data;
request->headers_out.content_length_n = range_end - range_start + 1;
}
if (contenttype != NULL) {
request->headers_out.content_type.len = strlen(contenttype);
request->headers_out.content_type.data = (u_char*)contenttype;
}
else ngx_http_set_content_type(request);
// use md5 field as ETag if possible
if (md5 != NULL) {
request->headers_out.etag = ngx_list_push(&request->headers_out.headers);
request->headers_out.etag->hash = 1;
request->headers_out.etag->key.len = sizeof("ETag") - 1;
request->headers_out.etag->key.data = (u_char*)"ETag";
ngx_buf_t *b;
b = ngx_create_temp_buf(request->pool, strlen(md5) + 2);
b->last = ngx_sprintf(b->last, "\"%s\"", md5);
request->headers_out.etag->value.len = strlen(md5) + 2;
request->headers_out.etag->value.data = b->start;
}
// use uploadDate field as last_modified if possible
if (last_modified) {
request->headers_out.last_modified_time = (time_t)(last_modified/1000);
}
/* Determine if content is gzipped, set headers accordingly */
if ( gridfile_get_boolean(&gfile,"gzipped") ) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, gridfile_get_field(&gfile,"gzipped") );
request->headers_out.content_encoding = ngx_list_push(&request->headers_out.headers);
if (request->headers_out.content_encoding == NULL) {
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return NGX_ERROR;
}
request->headers_out.content_encoding->hash = 1;
request->headers_out.content_encoding->key.len = sizeof("Content-Encoding") - 1;
request->headers_out.content_encoding->key.data = (u_char *) "Content-Encoding";
request->headers_out.content_encoding->value.len = sizeof("gzip") - 1;
request->headers_out.content_encoding->value.data = (u_char *) "gzip";
}
ngx_http_send_header(request);
// ---------- SEND THE BODY ---------- //
/* Empty file */
if (numchunks == 0) {
/* Allocate space for the response buffer */
buffer = ngx_pcalloc(request->pool, sizeof(ngx_buf_t));
if (buffer == NULL) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Failed to allocate response buffer");
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
buffer->pos = NULL;
buffer->last = NULL;
buffer->memory = 1;
buffer->last_buf = 1;
out.buf = buffer;
out.next = NULL;
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return ngx_http_output_filter(request, &out);
}
cursors = (mongo_cursor **)ngx_pcalloc(request->pool, sizeof(mongo_cursor *) * numchunks);
if (cursors == NULL) {
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
ngx_memzero( cursors, sizeof(mongo_cursor *) * numchunks);
/* Hook in the cleanup function */
gridfs_cln = ngx_pool_cleanup_add(request->pool, sizeof(ngx_http_gridfs_cleanup_t));
if (gridfs_cln == NULL) {
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
gridfs_cln->handler = ngx_http_gridfs_cleanup;
gridfs_clndata = gridfs_cln->data;
gridfs_clndata->cursors = cursors;
gridfs_clndata->numchunks = numchunks;
/* Read and serve chunk by chunk */
for (i = 0; i < numchunks; i++) {
/* Allocate space for the response buffer */
buffer = ngx_pcalloc(request->pool, sizeof(ngx_buf_t));
if (buffer == NULL) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Failed to allocate response buffer");
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
/* Fetch the chunk from mongo */
do {
e = FALSE;
cursors[i] = gridfile_get_chunks(&gfile, i, 1);
if (!(cursors[i] && mongo_cursor_next(cursors[i]) == MONGO_OK)) {
e = TRUE; ecounter++;
if (ecounter > MONGO_MAX_RETRIES_PER_REQUEST
|| ngx_http_mongo_reconnect(request->connection->log, mongo_conn) == NGX_ERROR
|| ngx_http_mongo_reauth(request->connection->log, mongo_conn) == NGX_ERROR) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Mongo connection dropped, could not reconnect");
if(mongo_conn->conn.connected) { mongo_disconnect(&mongo_conn->conn); }
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return NGX_HTTP_SERVICE_UNAVAILABLE;
}
}
} while (e);
chunk = cursors[i]->current;
bson_find(&it, &chunk, "data");
chunk_len = bson_iterator_bin_len( &it ); // break ngx_http_gridfs_module.c:1099
chunk_data = bson_iterator_bin_data( &it );
if (range_start == 0 && range_end == 0) {
/* <<no range request>> */
/* Set up the buffer chain */
buffer->pos = (u_char*)chunk_data;
buffer->last = (u_char*)chunk_data + chunk_len;
buffer->memory = 1;
buffer->last_buf = (i == numchunks-1);
out.buf = buffer;
out.next = NULL;
/* Serve the Chunk */
rc = ngx_http_output_filter(request, &out);
} else {
/* <<range request>> */
if ( range_start >= (current_buf_pos+chunk_len) ||
range_end <= current_buf_pos) {
/* no output */
ngx_pfree(request->pool, buffer);
} else {
if (range_start <= current_buf_pos) {
buffer->pos = (u_char*)chunk_data;
} else {
buffer->pos = (u_char*)chunk_data + (range_start - current_buf_pos);
}
if (range_end < (current_buf_pos+chunk_len)) {
buffer->last = (u_char*)chunk_data + (range_end - current_buf_pos + 1);
} else {
buffer->last = (u_char*)chunk_data + chunk_len;
}
if (buffer->pos == buffer->last) {
ngx_log_error(NGX_LOG_ALERT, request->connection->log, 0,
"zero size buf in writer "
"range_start:%d range_end:%d "
"current_buf_pos:%d chunk_len:%d i:%d numchunk:%d",
range_start,range_end,
current_buf_pos, chunk_len,
i,numchunks);
}
buffer->memory = 1;
buffer->last_buf = (i == numchunks-1) || (range_end < (current_buf_pos+chunk_len));
out.buf = buffer;
out.next = NULL;
/* Serve the Chunk */
rc = ngx_http_output_filter(request, &out);
}
}
current_buf_pos += chunk_len;
/* TODO: More Codes to Catch? */
if (rc == NGX_ERROR) {
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return NGX_ERROR;
}
}
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return rc;
}
static ngx_int_t ngx_http_gridfs_handler(ngx_http_request_t* request) {
ngx_http_gridfs_loc_conf_t* gridfs_conf;
ngx_http_core_loc_conf_t* core_conf;
ngx_buf_t* buffer;
ngx_chain_t out;
ngx_str_t location_name;
ngx_str_t full_uri;
// --------------- xulin add start -------------------
char* ml_args;
char* arg;
unsigned int add_arg;
unsigned int add_len;
// --------------- xulin add end -------------------
char* value;
ngx_http_mongo_connection_t *mongo_conn;
gridfs gfs;
gridfile gfile;
gridfs_offset length;
ngx_uint_t numchunks;
char* contenttype;
char* md5;
bson_date_t last_modified;
volatile ngx_uint_t i;
ngx_int_t rc = NGX_OK;
bson query;
bson_oid_t oid;
mongo_cursor ** cursors;
gridfs_offset chunk_len;
const char * chunk_data;
bson_iterator it;
bson chunk;
ngx_pool_cleanup_t* gridfs_cln;
ngx_http_gridfs_cleanup_t* gridfs_clndata;
int status;
volatile ngx_uint_t e = FALSE;
volatile ngx_uint_t ecounter = 0;
gridfs_conf = ngx_http_get_module_loc_conf(request, ngx_http_gridfs_module);
core_conf = ngx_http_get_module_loc_conf(request, ngx_http_core_module);
// ---------- ENSURE MONGO CONNECTION ---------- //
mongo_conn = ngx_http_get_mongo_connection( gridfs_conf->mongo );
if (mongo_conn == NULL) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Mongo Connection not found: \"%V\"", &gridfs_conf->mongo);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
if ( !(&mongo_conn->conn.connected)
&& (ngx_http_mongo_reconnect(request->connection->log, mongo_conn) == NGX_ERROR
|| ngx_http_mongo_reauth(request->connection->log, mongo_conn) == NGX_ERROR)) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Could not connect to mongo: \"%V\"", &gridfs_conf->mongo);
if(&mongo_conn->conn.connected) { mongo_disconnect(&mongo_conn->conn); }
return NGX_HTTP_SERVICE_UNAVAILABLE;
}
// ---------- RETRIEVE KEY ---------- //
location_name = core_conf->name;
full_uri = request->uri;
if (full_uri.len < location_name.len) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Invalid location name or uri.");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
value = (char*)malloc(sizeof(char) * (full_uri.len - location_name.len + 1 + request->args.len + 1));
if (value == NULL) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Failed to allocate memory for value buffer.");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
memcpy(value, full_uri.data + location_name.len, full_uri.len - location_name.len);
// ------------------------------------ xulin add start --------------------------------------
if (request->args.len > 0)
{
ml_args = (char*)malloc(sizeof(char) * (request->args.len + 1));
memcpy(ml_args, request->args.data, request->args.len);
ml_args[request->args.len] = '\0';
add_len = full_uri.len - location_name.len;
memcpy(value + add_len, "?", 1);
add_len += 1;
arg = strtok(ml_args, "&");
while (arg != NULL)
{
add_arg = 1;
if (strstr(arg, "xc_md5") != NULL)
{
add_arg = 0;
}
else if (strstr(arg, "_xingcloud_t") != NULL)
{
add_arg = 0;
}
if (add_arg == 1)
{
memcpy(value + add_len, arg, strlen(arg));
add_len += strlen(arg);
memcpy(value + add_len, "&", 1);
add_len += 1;
}
arg = strtok(NULL, "&");
}
free(ml_args);
if (value[add_len - 1] == '?' || value[add_len - 1] == '&')
{
value[add_len - 1] = '\0';
}
else
{
value[add_len] = '\0';
}
}
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "ml_url = [%s]", value);
// ------------------------------------ xulin add end --------------------------------------
if (!url_decode(value)) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Malformed request.");
free(value);
return NGX_HTTP_BAD_REQUEST;
}
// ---------- RETRIEVE GRIDFILE ---------- //
do {
e = FALSE;
if (gridfs_init(&mongo_conn->conn,
(const char*)gridfs_conf->db.data,
(const char*)gridfs_conf->root_collection.data,
&gfs) != MONGO_OK) {
e = TRUE; ecounter++;
if (ecounter > MONGO_MAX_RETRIES_PER_REQUEST
|| ngx_http_mongo_reconnect(request->connection->log, mongo_conn) == NGX_ERROR
|| ngx_http_mongo_reauth(request->connection->log, mongo_conn) == NGX_ERROR) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Mongo connection dropped, could not reconnect");
if(&mongo_conn->conn.connected) { mongo_disconnect(&mongo_conn->conn); }
free(value);
return NGX_HTTP_SERVICE_UNAVAILABLE;
}
}
} while (e);
bson_init(&query);
switch (gridfs_conf->type) {
case BSON_OID:
bson_oid_from_string(&oid, value);
bson_append_oid(&query, (char*)gridfs_conf->field.data, &oid);
break;
case BSON_INT:
bson_append_int(&query, (char*)gridfs_conf->field.data, ngx_atoi((u_char*)value, strlen(value)));
break;
case BSON_STRING:
bson_append_string(&query, (char*)gridfs_conf->field.data, value);
break;
}
bson_finish(&query);
status = gridfs_find_query(&gfs, &query, &gfile);
bson_destroy(&query);
free(value);
if(status == MONGO_ERROR) {
gridfs_destroy(&gfs);
return NGX_HTTP_NOT_FOUND;
}
/* Get information about the file */
length = gridfile_get_contentlength(&gfile);
numchunks = gridfile_get_numchunks(&gfile);
contenttype = (char*)gridfile_get_contenttype(&gfile);
md5 = (char*)gridfile_get_md5(&gfile);
last_modified = gridfile_get_uploaddate(&gfile);
// ---------- SEND THE HEADERS ---------- //
request->headers_out.status = NGX_HTTP_OK;
request->headers_out.content_length_n = length;
if (contenttype != NULL) {
request->headers_out.content_type.len = strlen(contenttype);
request->headers_out.content_type.data = (u_char*)contenttype;
}
else ngx_http_set_content_type(request);
// use md5 field as ETag if possible
if (md5 != NULL) {
request->headers_out.etag = ngx_list_push(&request->headers_out.headers);
request->headers_out.etag->hash = 1;
request->headers_out.etag->key.len = sizeof("ETag") - 1;
request->headers_out.etag->key.data = (u_char*)"ETag";
ngx_buf_t *b;
b = ngx_create_temp_buf(request->pool, strlen(md5) + 2);
b->last = ngx_sprintf(b->last, "\"%s\"", md5);
request->headers_out.etag->value.len = strlen(md5) + 2;
request->headers_out.etag->value.data = b->start;
}
// use uploadDate field as last_modified if possible
if (last_modified) {
request->headers_out.last_modified_time = (time_t)(last_modified/1000);
}
/* Determine if content is gzipped, set headers accordingly */
if ( gridfile_get_boolean(&gfile,"gzipped") ) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, gridfile_get_field(&gfile,"gzipped") );
request->headers_out.content_encoding = ngx_list_push(&request->headers_out.headers);
if (request->headers_out.content_encoding == NULL) {
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return NGX_ERROR;
}
request->headers_out.content_encoding->hash = 1;
request->headers_out.content_encoding->key.len = sizeof("Content-Encoding") - 1;
request->headers_out.content_encoding->key.data = (u_char *) "Content-Encoding";
request->headers_out.content_encoding->value.len = sizeof("gzip") - 1;
request->headers_out.content_encoding->value.data = (u_char *) "gzip";
}
ngx_http_send_header(request);
// ---------- SEND THE BODY ---------- //
/* Empty file */
if (numchunks == 0) {
/* Allocate space for the response buffer */
buffer = ngx_pcalloc(request->pool, sizeof(ngx_buf_t));
if (buffer == NULL) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Failed to allocate response buffer");
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
buffer->pos = NULL;
buffer->last = NULL;
buffer->memory = 1;
buffer->last_buf = 1;
out.buf = buffer;
out.next = NULL;
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return ngx_http_output_filter(request, &out);
}
cursors = (mongo_cursor **)ngx_pcalloc(request->pool, sizeof(mongo_cursor *) * numchunks);
if (cursors == NULL) {
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
ngx_memzero( cursors, sizeof(mongo_cursor *) * numchunks);
/* Hook in the cleanup function */
gridfs_cln = ngx_pool_cleanup_add(request->pool, sizeof(ngx_http_gridfs_cleanup_t));
if (gridfs_cln == NULL) {
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
gridfs_cln->handler = ngx_http_gridfs_cleanup;
gridfs_clndata = gridfs_cln->data;
gridfs_clndata->cursors = cursors;
gridfs_clndata->numchunks = numchunks;
/* Read and serve chunk by chunk */
for (i = 0; i < numchunks; i++) {
/* Allocate space for the response buffer */
buffer = ngx_pcalloc(request->pool, sizeof(ngx_buf_t));
if (buffer == NULL) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Failed to allocate response buffer");
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
/* Fetch the chunk from mongo */
do {
e = FALSE;
cursors[i] = gridfile_get_chunks(&gfile, i, 1);
if (!(cursors[i] && mongo_cursor_next(cursors[i]) == MONGO_OK)) {
e = TRUE; ecounter++;
if (ecounter > MONGO_MAX_RETRIES_PER_REQUEST
|| ngx_http_mongo_reconnect(request->connection->log, mongo_conn) == NGX_ERROR
|| ngx_http_mongo_reauth(request->connection->log, mongo_conn) == NGX_ERROR) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Mongo connection dropped, could not reconnect");
if(&mongo_conn->conn.connected) { mongo_disconnect(&mongo_conn->conn); }
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return NGX_HTTP_SERVICE_UNAVAILABLE;
}
}
} while (e);
chunk = cursors[i]->current;
bson_find(&it, &chunk, "data");
chunk_len = bson_iterator_bin_len( &it );
chunk_data = bson_iterator_bin_data( &it );
/* Set up the buffer chain */
buffer->pos = (u_char*)chunk_data;
buffer->last = (u_char*)chunk_data + chunk_len;
buffer->memory = 1;
buffer->last_buf = (i == numchunks-1);
out.buf = buffer;
out.next = NULL;
/* Serve the Chunk */
rc = ngx_http_output_filter(request, &out);
/* TODO: More Codes to Catch? */
if (rc == NGX_ERROR) {
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return NGX_ERROR;
}
}
gridfile_destroy(&gfile);
gridfs_destroy(&gfs);
return rc;
}
void test_random_write() {
mongo conn[1];
gridfs gfs[1];
gridfile* gfile;
char *data_before = (char*)bson_malloc( UPPER );
char *random_data = (char*)bson_malloc( UPPER );
char *buf = (char*) bson_malloc( UPPER );
int64_t i;
FILE *fd;
srand((unsigned int) time( NULL ) );
INIT_SOCKETS_FOR_WINDOWS;
if ( mongo_connect( conn, TEST_SERVER, 27017 ) ) {
printf( "failed to connect 2\n" );
exit( 1 );
}
gridfs_init( conn, "test", "fs", gfs );
fill_buffer_randomly( data_before, UPPER );
fill_buffer_randomly( random_data, UPPER );
for ( i = LOWER; i <= UPPER; i += DELTA ) {
int64_t j = i / 2 - 3;
int n, bytes_to_write_first;
/* Input from buffer */
gridfs_store_buffer( gfs, data_before, i, "input-buffer", "text/html", GRIDFILE_DEFAULT );
if ( i > DEFAULT_CHUNK_SIZE * 4 ) {
n = DEFAULT_CHUNK_SIZE * 3 + 6;
memcpy(&data_before[j], random_data, n); // Let's overwrite the buffer with bytes crossing multiple chunks
bytes_to_write_first = 10;
} else {
n = 6;
memcpy(random_data, "123456", n);
strncpy(&data_before[j], random_data, n); // Let's overwrite the buffer with a few some bytes
bytes_to_write_first = 0;
}
gfile = gridfile_create();
ASSERT(gridfs_find_filename(gfs, "input-buffer", gfile) == 0);
gridfile_writer_init(gfile, gfs, "input-buffer", "text/html", GRIDFILE_DEFAULT );
gridfile_seek(gfile, j); // Seek into the same buffer position within the GridFS file
if ( bytes_to_write_first ) {
gridfile_write_buffer(gfile, random_data, bytes_to_write_first); // Let's write 10 bytes first, and later the rest
}
gridfile_write_buffer(gfile, &random_data[bytes_to_write_first], n - bytes_to_write_first); // Try to write to the existing GridFS file on the position given by j
gridfile_seek(gfile, j);
gridfile_read( gfile, n, buf );
ASSERT(memcmp( buf, &data_before[j], n) == 0);
gridfile_writer_done(gfile);
ASSERT(gfile->pos == j + n);
gridfile_dispose(gfile);
test_gridfile( gfs, data_before, j + n > i ? j + n : i, "input-buffer", "text/html" );
/* Input from file */
fd = fopen( "input-file", "w" );
fwrite( data_before, sizeof( char ), (size_t) (j + n > i ? j + n : i), fd );
fclose( fd );
gridfs_store_file( gfs, "input-file", "input-file", "text/html", GRIDFILE_DEFAULT );
test_gridfile( gfs, data_before, j + n > i ? j + n : i, "input-file", "text/html" );
}
gridfs_destroy( gfs );
mongo_disconnect( conn );
mongo_destroy( conn );
free( data_before );
free( random_data );
free( buf );
/* Clean up files. */
_unlink( "input-file" );
_unlink( "output" );
}
void test_large( void ) {
mongo conn[1];
gridfs gfs[1];
gridfile gfile[1];
FILE *fd;
size_t i, n;
char *buffer = (char*)bson_malloc( LARGE );
char *read_buf = (char*)bson_malloc( LARGE );
int64_t filesize = ( int64_t )1024 * ( int64_t )LARGE;
mongo_write_concern wc;
bson lastError;
bson lastErrorCmd;
srand( (unsigned int) time( NULL ) );
INIT_SOCKETS_FOR_WINDOWS;
if ( mongo_client( conn, TEST_SERVER, 27017 ) ) {
printf( "failed to connect 1\n" );
exit( 1 );
}
mongo_write_concern_init(&wc);
wc.j = 1;
mongo_write_concern_finish(&wc);
mongo_set_write_concern(conn, &wc);
gridfs_init( conn, "test", "fs", gfs );
fd = fopen( "bigfile", "r" );
if( fd ) {
fclose( fd );
} else {
/* Create a very large file */
fill_buffer_randomly( buffer, ( int64_t )LARGE );
fd = fopen( "bigfile", "w" );
for( i=0; i<1024; i++ ) {
fwrite( buffer, 1, LARGE, fd );
}
fclose( fd );
}
/* Now read the file into GridFS */
gridfs_remove_filename( gfs, "bigfile" );
gridfs_store_file( gfs, "bigfile", "bigfile", "text/html", GRIDFILE_NOMD5 | GRIDFILE_COMPRESS);
gridfs_find_filename( gfs, "bigfile", gfile );
ASSERT( strcmp( gridfile_get_filename( gfile ), "bigfile" ) == 0 );
ASSERT( gridfile_get_contentlength( gfile ) == filesize );
fd = fopen( "bigfile", "r" );
while( ( n = fread( buffer, 1, MEDIUM, fd ) ) != 0 ) {
ASSERT( gridfile_read( gfile, MEDIUM, read_buf ) == n );
ASSERT( memcmp( buffer, read_buf, n ) == 0 );
}
fclose( fd );
gridfile_destroy( gfile );
/* Read the file using the streaming interface */
gridfs_remove_filename( gfs, "bigfile" );
gridfs_remove_filename( gfs, "bigfile-stream" );
gridfile_writer_init( gfile, gfs, "bigfile-stream", "text/html", GRIDFILE_NOMD5 | GRIDFILE_COMPRESS );
mongo_write_concern_destroy( &wc );
mongo_write_concern_init(&wc);
wc.j = 0; /* Let's reset write concern j field to zero, we will manually call getLastError with j = 1 */
mongo_write_concern_finish(&wc);
mongo_set_write_concern(conn, &wc);
fd = fopen( "bigfile", "r" );
i = 0;
while( ( n = fread( buffer, 1, READ_WRITE_BUF_SIZE, fd ) ) != 0 ) {
gridfile_write_buffer( gfile, buffer, n );
if(i++ % 10 == 0) {
bson_init( &lastErrorCmd );
bson_append_int( &lastErrorCmd, "getLastError", 1);
bson_append_int( &lastErrorCmd, "j", 1);
bson_finish( &lastErrorCmd );
bson_init( &lastError );
mongo_run_command( conn, "test", &lastErrorCmd, &lastError );
bson_destroy( &lastError );
bson_destroy( &lastErrorCmd );
}
}
mongo_write_concern_destroy( &wc );
mongo_write_concern_init(&wc);
wc.j = 1; /* Let's reset write concern j field to 1 */
mongo_write_concern_finish(&wc);
mongo_set_write_concern(conn, &wc);
fclose( fd );
gridfile_writer_done( gfile );
gridfs_find_filename( gfs, "bigfile-stream", gfile );
ASSERT( strcmp( gridfile_get_filename( gfile ), "bigfile-stream" ) == 0 );
ASSERT( gridfile_get_contentlength( gfile ) == filesize );
gridfs_remove_filename( gfs, "bigfile-stream" );
gridfs_destroy( gfs );
mongo_disconnect( conn );
mongo_destroy( conn );
bson_free( buffer );
bson_free( read_buf );
mongo_write_concern_destroy( &wc );
}
static ngx_int_t ngx_http_gridfs_handler(ngx_http_request_t* request) {
ngx_http_gridfs_loc_conf_t* gridfs_conf;
ngx_http_core_loc_conf_t* core_conf;
ngx_buf_t* buffer;
ngx_chain_t out;
ngx_str_t location_name;
ngx_str_t full_uri;
char* value;
gridfs gfs;
gridfile gfile;
gridfs_offset length;
ngx_uint_t chunksize;
ngx_uint_t numchunks;
char* contenttype;
ngx_uint_t i;
ngx_int_t rc = NGX_OK;
bson query;
bson_buffer buf;
bson_oid_t oid;
mongo_cursor ** cursors;
gridfs_offset chunk_len;
const char * chunk_data;
bson_iterator it;
bson chunk;
ngx_pool_cleanup_t* gridfs_cln;
ngx_http_gridfs_cleanup_t* gridfs_clndata;
gridfs_conf = ngx_http_get_module_loc_conf(request, ngx_http_gridfs_module);
core_conf = ngx_http_get_module_loc_conf(request, ngx_http_core_module);
location_name = core_conf->name;
full_uri = request->uri;
gridfs_cln = ngx_pool_cleanup_add(request->pool, sizeof(ngx_http_gridfs_cleanup_t));
if (gridfs_cln == NULL) {
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
gridfs_cln->handler = ngx_http_gridfs_cleanup;
gridfs_clndata = gridfs_cln->data;
/* defensive */
if (full_uri.len < location_name.len) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Invalid location name or uri.");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
/* Extract the value from the uri */
value = (char*)malloc(sizeof(char) * (full_uri.len - location_name.len + 1));
if (value == NULL) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Failed to allocate memory for value buffer.");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
memcpy(value, full_uri.data + location_name.len, full_uri.len - location_name.len);
value[full_uri.len - location_name.len] = '\0';
/* URL Decoding */
if (!url_decode(value)) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Malformed request.");
free(value);
return NGX_HTTP_BAD_REQUEST;
}
/* Find the GridFile */
gridfs_init(gridfs_conf->mongod_conn,
(const char*)gridfs_conf->gridfs_db.data,
(const char*)gridfs_conf->gridfs_root_collection.data,
&gfs);
bson_buffer_init(&buf);
switch (gridfs_conf->gridfs_type) {
case bson_oid:
bson_oid_from_string(&oid, value);
bson_append_oid(&buf, (char*)gridfs_conf->gridfs_field.data, &oid);
break;
case bson_int:
bson_append_int(&buf, (char*)gridfs_conf->gridfs_field.data, ngx_atoi((u_char*)value, strlen(value)));
break;
case bson_string:
bson_append_string(&buf, (char*)gridfs_conf->gridfs_field.data, value);
break;
}
bson_from_buffer(&query, &buf);
if(!gridfs_find_query(&gfs, &query, &gfile)){
bson_destroy(&query);
free(value);
return NGX_HTTP_NOT_FOUND;
}
bson_destroy(&query);
free(value);
/* Get information about the file */
length = gridfile_get_contentlength(&gfile);
chunksize = gridfile_get_chunksize(&gfile);
numchunks = gridfile_get_numchunks(&gfile);
contenttype = (char*)gridfile_get_contenttype(&gfile);
/* Set the headers */
request->headers_out.status = NGX_HTTP_OK;
request->headers_out.content_length_n = length;
if (contenttype != NULL) {
request->headers_out.content_type.len = strlen(contenttype);
request->headers_out.content_type.data = (u_char*)contenttype;
}
else ngx_http_set_content_type(request);
/* Determine if content is gzipped, set headers accordingly */
if ( gridfile_get_boolean(&gfile,"gzipped") ) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, gridfile_get_field(&gfile,"gzipped") );
request->headers_out.content_encoding = ngx_list_push(&request->headers_out.headers);
if (request->headers_out.content_encoding == NULL) {
return NGX_ERROR;
}
request->headers_out.content_encoding->hash = 1;
request->headers_out.content_encoding->key.len = sizeof("Content-Encoding") - 1;
request->headers_out.content_encoding->key.data = (u_char *) "Content-Encoding";
request->headers_out.content_encoding->value.len = sizeof("gzip") - 1;
request->headers_out.content_encoding->value.data = (u_char *) "gzip";
}
ngx_http_send_header(request);
if (numchunks == 0) {
/* Allocate space for the response buffer */
buffer = ngx_pcalloc(request->pool, sizeof(ngx_buf_t));
if (buffer == NULL) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Failed to allocate response buffer");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
buffer->pos = NULL;
buffer->last = NULL;
buffer->memory = 1;
buffer->last_buf = 1;
out.buf = buffer;
out.next = NULL;
return ngx_http_output_filter(request, &out);
}
cursors = (mongo_cursor **)ngx_pcalloc(request->pool, sizeof(mongo_cursor *) * numchunks);
/* Read and serve chunk by chunk */
for (i = 0; i < numchunks; i++) {
/* Allocate space for the response buffer */
buffer = ngx_pcalloc(request->pool, sizeof(ngx_buf_t));
if (buffer == NULL) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Failed to allocate response buffer");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
/* Fetch the chunk from mongo */
cursors[i] = gridfile_get_chunks(&gfile, i, 1);
mongo_cursor_next(cursors[i]);
chunk = cursors[i]->current;
bson_find(&it, &chunk, "data");
chunk_len = bson_iterator_bin_len( &it );
chunk_data = bson_iterator_bin_data( &it );
/* Set up the buffer chain */
buffer->pos = (u_char*)chunk_data;
buffer->last = (u_char*)chunk_data + chunk_len;
buffer->memory = 1;
buffer->last_buf = (i == numchunks-1);
out.buf = buffer;
out.next = NULL;
/* Serve the Chunk */
rc = ngx_http_output_filter(request, &out);
/* TODO: More Codes to Catch? */
if (rc == NGX_ERROR) {
return NGX_ERROR;
}
}
gridfs_clndata->cursors = cursors;
gridfs_clndata->numchunks = numchunks;
return rc;
}
static ngx_int_t ngx_http_gridfs_handler(ngx_http_request_t* request) {
ngx_http_gridfs_loc_conf_t* gridfs_conf;
ngx_http_core_loc_conf_t* core_conf;
ngx_buf_t* buffer;
ngx_chain_t out;
ngx_str_t location_name;
ngx_str_t full_uri;
char* value;
gridfs gfs;
gridfile gfile;
gridfs_offset length;
char* data;
ngx_uint_t chunksize;
ngx_uint_t numchunks;
ngx_uint_t chunklength;
char* contenttype;
ngx_uint_t i;
ngx_int_t rc = NGX_OK;
bson query;
bson_buffer buf;
bson_oid_t oid;
gridfs_conf = ngx_http_get_module_loc_conf(request, ngx_http_gridfs_module);
core_conf = ngx_http_get_module_loc_conf(request, ngx_http_core_module);
location_name = core_conf->name;
full_uri = request->uri;
/* defensive */
if (full_uri.len < location_name.len) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Invalid location name or uri.");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
/* Extract the value from the uri */
value = (char*)malloc(sizeof(char) * (full_uri.len - location_name.len + 1));
if (value == NULL) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Failed to allocate memory for value buffer.");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
memcpy(value, full_uri.data + location_name.len, full_uri.len - location_name.len);
value[full_uri.len - location_name.len] = '\0';
/* URL Decoding */
if (!url_decode(value)) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Malformed request.");
free(value);
return NGX_HTTP_BAD_REQUEST;
}
/* If no mongo connection, create a default connection */
/* TODO: Find a better place for this logic */
if (!gridfs_conf->gridfs_conn->connected) {
switch (mongo_connect(gridfs_conf->gridfs_conn, NULL)) {
case mongo_conn_success:
break;
case mongo_conn_bad_arg:
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Mongo Exception: Bad Arguments");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
case mongo_conn_no_socket:
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Mongo Exception: No Socket");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
case mongo_conn_fail:
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Mongo Exception: Connection Failure");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
case mongo_conn_not_master:
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Mongo Exception: Not Master");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
default:
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Mongo Exception: Unknown Error");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
}
/* Find the GridFile */
gridfs_init(gridfs_conf->gridfs_conn,
(const char*)gridfs_conf->gridfs_db.data,
(const char*)gridfs_conf->gridfs_root_collection.data,
&gfs);
bson_buffer_init(&buf);
switch (gridfs_conf->gridfs_type) {
case bson_oid:
bson_oid_from_string(&oid, value);
bson_append_oid(&buf, (char*)gridfs_conf->gridfs_field.data, &oid);
break;
case bson_int:
bson_append_int(&buf, (char*)gridfs_conf->gridfs_field.data, atoi(value));
break;
case bson_string:
bson_append_string(&buf, (char*)gridfs_conf->gridfs_field.data, value);
break;
}
bson_from_buffer(&query, &buf);
if(!gridfs_find_query(&gfs, &query, &gfile)){
bson_destroy(&query);
free(value);
return NGX_HTTP_NOT_FOUND;
}
bson_destroy(&query);
free(value);
/* Get information about the file */
length = gridfile_get_contentlength(&gfile);
chunksize = gridfile_get_chunksize(&gfile);
numchunks = gridfile_get_numchunks(&gfile);
contenttype = (char*)gridfile_get_contenttype(&gfile);
/* Set the headers */
request->headers_out.status = NGX_HTTP_OK;
request->headers_out.content_length_n = length;
if (contenttype != NULL) {
request->headers_out.content_type.len = strlen(contenttype);
request->headers_out.content_type.data = (u_char*)contenttype;
}
else ngx_http_set_content_type(request);
ngx_http_send_header(request);
/* Read and serve chunk by chunk */
for (i = 0; i < numchunks; i++) {
/* Allocate space for the response buffer */
buffer = ngx_pcalloc(request->pool, sizeof(ngx_buf_t));
if (buffer == NULL) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Failed to allocate response buffer");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
/* Allocate space for the buffer of data */
data = ngx_pcalloc(request->pool, sizeof(char)*chunksize);
if (data == NULL) {
ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
"Failed to allocate buffer for data");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
/* Set up the buffer chain */
chunklength = gridfile_read(&gfile, chunksize, data);
buffer->pos = (u_char*)data;
buffer->last = (u_char*)data + chunklength;
buffer->memory = 1;
buffer->last_buf = (i == numchunks-1);
out.buf = buffer;
out.next = NULL;
/* Serve the Chunk */
rc = ngx_http_output_filter(request, &out);
/* TODO: More Codes to Catch? */
if (rc == NGX_ERROR) {
return NGX_ERROR;
}
}
return rc;
}
