static void test_delete( void ) { mongo conn[1]; gridfs gfs[1]; gridfile gfile[1]; char *data = (char*)bson_malloc( 1024 ); const char *testFile = "test-delete"; INIT_SOCKETS_FOR_WINDOWS; CONN_CLIENT_TEST; GFS_INIT; ASSERT( gridfs_store_buffer( gfs, data, 1024, testFile, "text/html", GRIDFILE_DEFAULT ) == MONGO_OK ); ASSERT( gridfs_find_filename( gfs, testFile, gfile ) == MONGO_OK ); gridfile_destroy( gfile ); ASSERT( gridfs_remove_filename( gfs, testFile ) == MONGO_OK ); ASSERT( gridfs_find_filename( gfs, testFile, gfile ) == MONGO_ERROR ); ASSERT( gridfs_find_filename( gfs, "bogus-file-does-not-exist", gfile ) == MONGO_ERROR ); ASSERT( gridfs_remove_filename( gfs, "bogus-file-does-not-exist" ) == MONGO_ERROR ); gridfs_destroy( gfs ); mongo_disconnect( conn ); mongo_destroy( conn ); bson_free( data ); }
void test_gridfile(gridfs *gfs, char *data_before, uint64_t length, char *filename, char *content_type) { gridfile gfile[1]; char *data_after = malloc( LARGE ); if( data_after == NULL ) { printf("Failed to allocated memory"); exit(1); } FILE * fd; mongo_md5_state_t pms[1]; mongo_md5_byte_t digest[16]; char hex_digest[33]; uint64_t i = length; int n; gridfs_find_filename(gfs, filename, gfile); ASSERT(gridfile_exists(gfile)); fd = fopen("output", "w+"); gridfile_write_file(gfile, fd); fseek(fd, 0, SEEK_SET); ASSERT(fread(data_after, length, sizeof(char), fd)); fclose(fd); ASSERT( strncmp(data_before, data_after, length) == 0 ); gridfile_read( gfile, length, data_after ); ASSERT( strncmp(data_before, data_after, length) == 0 ); ASSERT( strcmp( gridfile_get_filename( gfile ), filename ) == 0 ); ASSERT( gridfile_get_contentlength( gfile ) == length ); ASSERT( gridfile_get_chunksize( gfile ) == DEFAULT_CHUNK_SIZE ); ASSERT( strcmp( gridfile_get_contenttype( gfile ), content_type ) == 0) ; ASSERT( strncmp( data_before, data_after, length ) == 0 ); mongo_md5_init(pms); n = 0; while( i > INT_MAX ) { mongo_md5_append(pms, (const mongo_md5_byte_t *)data_before + (n * INT_MAX), INT_MAX); i -= INT_MAX; n += 1; } if( i > 0 ) mongo_md5_append(pms, (const mongo_md5_byte_t *)data_before + (n * INT_MAX), i); mongo_md5_finish(pms, digest); digest2hex(digest, hex_digest); ASSERT( strcmp( gridfile_get_md5( gfile ), hex_digest ) == 0 ); gridfile_destroy(gfile); gridfs_remove_filename(gfs, filename); free( data_after ); }
void test_gridfile(gridfs *gfs, char *data_before, size_t length, char *filename, char *content_type) { gridfile gfile[1]; char data_after[UPPER]; FILE * fd; mongo_md5_state_t pms[1]; mongo_md5_byte_t digest[16]; char hex_digest[33]; gridfs_find_filename(gfs, filename, gfile); ASSERT(gridfile_exists(gfile)); fd = fopen("output", "w+"); gridfile_write_file(gfile, fd); fseek(fd, 0, SEEK_SET); ASSERT(fread(data_after, length, sizeof(char), fd)); fclose(fd); ASSERT( strncmp(data_before, data_after, length) == 0 ); gridfile_read( gfile, length, data_after); ASSERT( strncmp(data_before, data_after, length) == 0 ); ASSERT( strcmp( gridfile_get_filename( gfile ), filename ) == 0 ); ASSERT( gridfile_get_contentlength( gfile ) == length ); ASSERT( gridfile_get_chunksize( gfile ) == DEFAULT_CHUNK_SIZE ); ASSERT( strcmp( gridfile_get_contenttype( gfile ), content_type ) == 0) ; mongo_md5_init(pms); mongo_md5_append(pms, (const mongo_md5_byte_t *)data_before, (int)length); mongo_md5_finish(pms, digest); digest2hex(digest, hex_digest); ASSERT( strcmp( gridfile_get_md5( gfile ), hex_digest ) == 0 ); gridfile_destroy(gfile); gridfs_remove_filename(gfs, filename); }
//! Read raster data from MongoDB int clsRasterData::ReadFromMongoDB(gridfs *gfs, const char* remoteFilename) { gridfile gfile[1]; bson b[1]; bson_init(b); bson_append_string(b, "filename", remoteFilename); bson_finish(b); int flag = gridfs_find_query(gfs, b, gfile); if(0 != flag) { throw ModelException("clsRasterData", "ReadFromMongoDB", "The file " + string(remoteFilename) + " does not exist."); } size_t length = (size_t)gridfile_get_contentlength(gfile); char* buf = (char*)malloc(length); gridfile_read (gfile, length, buf); float *data = (float*)buf; bson bmeta[1]; gridfile_get_metadata(gfile, bmeta); bson_iterator iterator[1]; if ( bson_find( iterator, bmeta, "NCOLS" )) m_headers["NCOLS"] = (float)bson_iterator_int(iterator); if ( bson_find( iterator, bmeta, "NROWS" )) m_headers["NROWS"] = (float)bson_iterator_int(iterator); if ( bson_find( iterator, bmeta, "NODATA_VALUE" )) m_headers["NODATA_VALUE"] = (float)bson_iterator_double(iterator); if ( bson_find( iterator, bmeta, "XLLCENTER" )) m_headers["XLLCENTER"] = (float)bson_iterator_double(iterator); if ( bson_find( iterator, bmeta, "YLLCENTER" )) m_headers["YLLCENTER"] = (float)bson_iterator_double(iterator); if ( bson_find( iterator, bmeta, "CELLSIZE" )) { m_headers["CELLSIZE"] = (float)bson_iterator_double(iterator); //m_headers["DY"] = m_headers["DX"]; } int nRows = (int)m_headers["NROWS"]; int nCols = (int)m_headers["NCOLS"]; vector<float> values; vector<int> positionRows; vector<int> positionCols; //get all valid values float nodataFloat = m_headers["NODATA_VALUE"]; for (int i = 0; i < nRows; ++i) { for (int j = 0; j < nCols; ++j) { int index = i*nCols + j; float value = data[index]; if(FloatEqual(nodataFloat, value)) continue; values.push_back(value); positionRows.push_back(i); positionCols.push_back(j); } } //create float array m_nRows = values.size(); m_rasterData = new float[m_nRows]; m_rasterPositionData = new float*[m_nRows]; for (int i = 0; i < m_nRows; ++i) { m_rasterData[i] = values.at(i); m_rasterPositionData[i] = new float[2]; m_rasterPositionData[i][0] = float(positionRows.at(i)); m_rasterPositionData[i][1] = float(positionCols.at(i)); } bson_destroy(b); gridfile_destroy(gfile); free(buf); return 0; }
void clsRasterData::outputToMongoDB(map<string,float> header, int nValid, float** position, float* value,string remoteFilename, gridfs* gfs) { float noData = -9999.0f; // prepare binary data int rows = int(header["NROWS"]); int cols = int(header["NCOLS"]); float *data = new float[rows*cols]; int index = 0; int dataIndex = 0; for (int i = 0; i < rows; ++i) { for (int j = 0; j < cols; ++j) { dataIndex = i*cols + j; if(index < nValid) { if(position[index][0] == i && position[index][1] == j) { data[dataIndex] = value[index]; index++; } else data[dataIndex] = noData; } else data[dataIndex] = noData; } } bson *p = (bson*)malloc(sizeof(bson)); bson_init(p); bson_append_string(p, "ID", remoteFilename.c_str()); bson_append_double(p, "CELLSIZE", header["CELLSIZE"]); bson_append_double(p, "NODATA_VALUE", noData); bson_append_double(p, "NCOLS", cols); bson_append_double(p, "NROWS", rows); bson_append_double(p, "XLLCENTER", header["XLLCENTER"]); bson_append_double(p, "YLLCENTER", header["YLLCENTER"]); bson_finish(p); gridfile gfile[1]; gridfile_writer_init(gfile, gfs, remoteFilename.c_str(), "float"); size_t iID = remoteFilename.find_first_of('_'); int subbasinID = atoi(remoteFilename.substr(0, iID).c_str()); gfile->id.ints[0] = subbasinID; for (int k = 0; k < rows; k++) { gridfile_write_buffer(gfile, (const char*)(data+cols*k), sizeof(float)*cols); } gridfile_set_metadata(gfile, p); int response = gridfile_writer_done(gfile); //cout << remoteFilename << "\t" << gfile->id.ints[0] << gfile->id.ints[1] << gfile->id.ints[2] << endl; gridfile_destroy(gfile); bson_destroy(p); free(p); delete data; }
//! Read raster data from MongoDB and select by mask int clsRasterData::ReadFromMongoDB(gridfs *gfs, const char* remoteFilename,clsRasterData* mask) { if(mask == NULL) ReadFromMongoDB(gfs, remoteFilename); else { //clock_t start = clock(); gridfile gfile[1]; bson b[1]; bson_init(b); bson_append_string(b, "filename", remoteFilename); bson_finish(b); int flag = gridfs_find_query(gfs, b, gfile); if(0 != flag) { //cout << "Failed in ReadFromMongoDB, Remote file: " << remoteFilename << endl; return -1; } size_t length = (size_t)gridfile_get_contentlength(gfile); char* buf = (char*)malloc(length); gridfile_read (gfile, length, buf); float *data = (float*)buf; //clock_t end = clock(); //cout << "Read data: " << end - start << endl; //start = clock(); //get the data posotion from mask int nRows; float** validPosition; mask->getRasterPositionData(&nRows,&validPosition); m_nRows = nRows; m_mask = mask; //set header m_headers["NCOLS"] = mask->getASCCols(); m_headers["NROWS"] = mask->getASCRows(); m_headers["NODATA_VALUE"] = mask->getNoDataValue(); m_headers["CELLSIZE"] = mask->getCellWidth(); //m_headers["DY"] = mask->getCellWidth(); m_headers["XLLCENTER"] = mask->getXllCenter(); m_headers["YLLCENTER"] = mask->getYllCenter(); //read data m_rasterData = new float[nRows]; int ascCols = mask->getASCCols(); int ascRows = mask->getASCRows(); //int index = 0; //for (int i = 0; i < ascRows; ++i) //{ // for (int j = 0; j < ascCols; ++j) // { // float value = data[i*ascCols + j]; // if(index < nRows) // { // if(validPosition[index][0] == i && validPosition[index][1] == j) // { // m_rasterData[index] = value; // index++; // } // } // } //} for(int index = 0; index < nRows; index++) { int i = validPosition[index][0]; int j = validPosition[index][1]; int rasterIndex = i*ascCols + j; m_rasterData[index] = data[rasterIndex]; } //end = clock(); //cout << "Rearrange data: " << end-start << endl; bson_destroy(b); gridfile_destroy(gfile); free(buf); } return 0; }
static ngx_int_t ngx_http_gridfs_handler(ngx_http_request_t* request) { ngx_http_gridfs_loc_conf_t* gridfs_conf; ngx_http_core_loc_conf_t* core_conf; ngx_buf_t* buffer; ngx_chain_t out; ngx_str_t location_name; ngx_str_t full_uri; char* value; ngx_http_mongo_connection_t *mongo_conn; gridfs gfs; gridfile gfile; gridfs_offset length; ngx_uint_t numchunks; char* contenttype; char* md5; bson_date_t last_modified; volatile ngx_uint_t i; ngx_int_t rc = NGX_OK; bson query; bson_oid_t oid; mongo_cursor ** cursors; gridfs_offset chunk_len; const char * chunk_data; bson_iterator it; bson chunk; ngx_pool_cleanup_t* gridfs_cln; ngx_http_gridfs_cleanup_t* gridfs_clndata; int status; volatile ngx_uint_t e = FALSE; volatile ngx_uint_t ecounter = 0; uint64_t range_start = 0; uint64_t range_end = 0; uint64_t current_buf_pos = 0; gridfs_conf = ngx_http_get_module_loc_conf(request, ngx_http_gridfs_module); core_conf = ngx_http_get_module_loc_conf(request, ngx_http_core_module); // ---------- ENSURE MONGO CONNECTION ---------- // mongo_conn = ngx_http_get_mongo_connection( gridfs_conf->mongo ); if (mongo_conn == NULL) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Mongo Connection not found: \"%V\"", &gridfs_conf->mongo); return NGX_HTTP_INTERNAL_SERVER_ERROR; } if (mongo_conn->conn.connected == 0) { if (ngx_http_mongo_reconnect(request->connection->log, mongo_conn) == NGX_ERROR) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Could not connect to mongo: \"%V\"", &gridfs_conf->mongo); if(mongo_conn->conn.connected) { mongo_disconnect(&mongo_conn->conn); } return NGX_HTTP_SERVICE_UNAVAILABLE; } if (ngx_http_mongo_reauth(request->connection->log, mongo_conn) == NGX_ERROR) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Failed to reauth to mongo: \"%V\"", &gridfs_conf->mongo); if(mongo_conn->conn.connected) { mongo_disconnect(&mongo_conn->conn); } return NGX_HTTP_SERVICE_UNAVAILABLE; } } // ---------- RETRIEVE KEY ---------- // location_name = core_conf->name; full_uri = request->uri; if (full_uri.len < location_name.len) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Invalid location name or uri."); return NGX_HTTP_INTERNAL_SERVER_ERROR; } value = (char*)malloc(sizeof(char) * (full_uri.len - location_name.len + 1)); if (value == NULL) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Failed to allocate memory for value buffer."); return NGX_HTTP_INTERNAL_SERVER_ERROR; } memcpy(value, full_uri.data + location_name.len, full_uri.len - location_name.len); value[full_uri.len - location_name.len] = '\0'; if (!url_decode(value)) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Malformed request."); free(value); return NGX_HTTP_BAD_REQUEST; } // ---------- RETRIEVE GRIDFILE ---------- // bson_init(&query); switch (gridfs_conf->type) { case BSON_OID: bson_oid_from_string(&oid, value); bson_append_oid(&query, (char*)gridfs_conf->field.data, &oid); break; case BSON_INT: bson_append_int(&query, (char*)gridfs_conf->field.data, ngx_atoi((u_char*)value, strlen(value))); break; case BSON_STRING: bson_append_string(&query, (char*)gridfs_conf->field.data, value); break; } bson_finish(&query); do { e = FALSE; if (gridfs_init(&mongo_conn->conn, (const char*)gridfs_conf->db.data, (const char*)gridfs_conf->root_collection.data, &gfs) != MONGO_OK || (status = gridfs_find_query(&gfs, &query, &gfile) == MONGO_ERROR)) { e = TRUE; ecounter++; if (ecounter > MONGO_MAX_RETRIES_PER_REQUEST || ngx_http_mongo_reconnect(request->connection->log, mongo_conn) == NGX_ERROR || ngx_http_mongo_reauth(request->connection->log, mongo_conn) == NGX_ERROR) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Mongo connection dropped, could not reconnect"); if(&mongo_conn->conn.connected) { mongo_disconnect(&mongo_conn->conn); } bson_destroy(&query); free(value); return NGX_HTTP_SERVICE_UNAVAILABLE; } } } while (e); bson_destroy(&query); free(value); /* Get information about the file */ length = gridfile_get_contentlength(&gfile); numchunks = gridfile_get_numchunks(&gfile); // NaN workaround if (numchunks > INT_MAX) { gridfile_destroy(&gfile); gridfs_destroy(&gfs); return NGX_HTTP_NOT_FOUND; } contenttype = (char*)gridfile_get_contenttype(&gfile); md5 = (char*)gridfile_get_md5(&gfile); last_modified = gridfile_get_uploaddate(&gfile); // ---------- Partial Range // set follow-fork-mode child // attach (pid) // break ngx_http_gridfs_module.c:959 if (request->headers_in.range) { gridfs_parse_range(request, &request->headers_in.range->value, &range_start, &range_end, length); } // ---------- SEND THE HEADERS ---------- // if (range_start == 0 && range_end == 0) { request->headers_out.status = NGX_HTTP_OK; request->headers_out.content_length_n = length; } else { request->headers_out.status = NGX_HTTP_PARTIAL_CONTENT; request->headers_out.content_length_n = length; //request->headers_out.content_range = range_end - range_start + 1; ngx_table_elt_t *content_range; content_range = ngx_list_push(&request->headers_out.headers); if (content_range == NULL) { return NGX_ERROR; } request->headers_out.content_range = content_range; content_range->hash = 1; ngx_str_set(&content_range->key, "Content-Range"); content_range->value.data = ngx_pnalloc(request->pool,sizeof("bytes -/") - 1 + 3 * NGX_OFF_T_LEN); if (content_range->value.data == NULL) { return NGX_ERROR; } /* "Content-Range: bytes SSSS-EEEE/TTTT" header */ content_range->value.len = ngx_sprintf(content_range->value.data, "bytes %O-%O/%O", range_start, range_end, request->headers_out.content_length_n) - content_range->value.data; request->headers_out.content_length_n = range_end - range_start + 1; } if (contenttype != NULL) { request->headers_out.content_type.len = strlen(contenttype); request->headers_out.content_type.data = (u_char*)contenttype; } else ngx_http_set_content_type(request); // use md5 field as ETag if possible if (md5 != NULL) { request->headers_out.etag = ngx_list_push(&request->headers_out.headers); request->headers_out.etag->hash = 1; request->headers_out.etag->key.len = sizeof("ETag") - 1; request->headers_out.etag->key.data = (u_char*)"ETag"; ngx_buf_t *b; b = ngx_create_temp_buf(request->pool, strlen(md5) + 2); b->last = ngx_sprintf(b->last, "\"%s\"", md5); request->headers_out.etag->value.len = strlen(md5) + 2; request->headers_out.etag->value.data = b->start; } // use uploadDate field as last_modified if possible if (last_modified) { request->headers_out.last_modified_time = (time_t)(last_modified/1000); } /* Determine if content is gzipped, set headers accordingly */ if ( gridfile_get_boolean(&gfile,"gzipped") ) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, gridfile_get_field(&gfile,"gzipped") ); request->headers_out.content_encoding = ngx_list_push(&request->headers_out.headers); if (request->headers_out.content_encoding == NULL) { gridfile_destroy(&gfile); gridfs_destroy(&gfs); return NGX_ERROR; } request->headers_out.content_encoding->hash = 1; request->headers_out.content_encoding->key.len = sizeof("Content-Encoding") - 1; request->headers_out.content_encoding->key.data = (u_char *) "Content-Encoding"; request->headers_out.content_encoding->value.len = sizeof("gzip") - 1; request->headers_out.content_encoding->value.data = (u_char *) "gzip"; } ngx_http_send_header(request); // ---------- SEND THE BODY ---------- // /* Empty file */ if (numchunks == 0) { /* Allocate space for the response buffer */ buffer = ngx_pcalloc(request->pool, sizeof(ngx_buf_t)); if (buffer == NULL) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Failed to allocate response buffer"); gridfile_destroy(&gfile); gridfs_destroy(&gfs); return NGX_HTTP_INTERNAL_SERVER_ERROR; } buffer->pos = NULL; buffer->last = NULL; buffer->memory = 1; buffer->last_buf = 1; out.buf = buffer; out.next = NULL; gridfile_destroy(&gfile); gridfs_destroy(&gfs); return ngx_http_output_filter(request, &out); } cursors = (mongo_cursor **)ngx_pcalloc(request->pool, sizeof(mongo_cursor *) * numchunks); if (cursors == NULL) { gridfile_destroy(&gfile); gridfs_destroy(&gfs); return NGX_HTTP_INTERNAL_SERVER_ERROR; } ngx_memzero( cursors, sizeof(mongo_cursor *) * numchunks); /* Hook in the cleanup function */ gridfs_cln = ngx_pool_cleanup_add(request->pool, sizeof(ngx_http_gridfs_cleanup_t)); if (gridfs_cln == NULL) { gridfile_destroy(&gfile); gridfs_destroy(&gfs); return NGX_HTTP_INTERNAL_SERVER_ERROR; } gridfs_cln->handler = ngx_http_gridfs_cleanup; gridfs_clndata = gridfs_cln->data; gridfs_clndata->cursors = cursors; gridfs_clndata->numchunks = numchunks; /* Read and serve chunk by chunk */ for (i = 0; i < numchunks; i++) { /* Allocate space for the response buffer */ buffer = ngx_pcalloc(request->pool, sizeof(ngx_buf_t)); if (buffer == NULL) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Failed to allocate response buffer"); gridfile_destroy(&gfile); gridfs_destroy(&gfs); return NGX_HTTP_INTERNAL_SERVER_ERROR; } /* Fetch the chunk from mongo */ do { e = FALSE; cursors[i] = gridfile_get_chunks(&gfile, i, 1); if (!(cursors[i] && mongo_cursor_next(cursors[i]) == MONGO_OK)) { e = TRUE; ecounter++; if (ecounter > MONGO_MAX_RETRIES_PER_REQUEST || ngx_http_mongo_reconnect(request->connection->log, mongo_conn) == NGX_ERROR || ngx_http_mongo_reauth(request->connection->log, mongo_conn) == NGX_ERROR) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Mongo connection dropped, could not reconnect"); if(mongo_conn->conn.connected) { mongo_disconnect(&mongo_conn->conn); } gridfile_destroy(&gfile); gridfs_destroy(&gfs); return NGX_HTTP_SERVICE_UNAVAILABLE; } } } while (e); chunk = cursors[i]->current; bson_find(&it, &chunk, "data"); chunk_len = bson_iterator_bin_len( &it ); // break ngx_http_gridfs_module.c:1099 chunk_data = bson_iterator_bin_data( &it ); if (range_start == 0 && range_end == 0) { /* <<no range request>> */ /* Set up the buffer chain */ buffer->pos = (u_char*)chunk_data; buffer->last = (u_char*)chunk_data + chunk_len; buffer->memory = 1; buffer->last_buf = (i == numchunks-1); out.buf = buffer; out.next = NULL; /* Serve the Chunk */ rc = ngx_http_output_filter(request, &out); } else { /* <<range request>> */ if ( range_start >= (current_buf_pos+chunk_len) || range_end <= current_buf_pos) { /* no output */ ngx_pfree(request->pool, buffer); } else { if (range_start <= current_buf_pos) { buffer->pos = (u_char*)chunk_data; } else { buffer->pos = (u_char*)chunk_data + (range_start - current_buf_pos); } if (range_end < (current_buf_pos+chunk_len)) { buffer->last = (u_char*)chunk_data + (range_end - current_buf_pos + 1); } else { buffer->last = (u_char*)chunk_data + chunk_len; } if (buffer->pos == buffer->last) { ngx_log_error(NGX_LOG_ALERT, request->connection->log, 0, "zero size buf in writer " "range_start:%d range_end:%d " "current_buf_pos:%d chunk_len:%d i:%d numchunk:%d", range_start,range_end, current_buf_pos, chunk_len, i,numchunks); } buffer->memory = 1; buffer->last_buf = (i == numchunks-1) || (range_end < (current_buf_pos+chunk_len)); out.buf = buffer; out.next = NULL; /* Serve the Chunk */ rc = ngx_http_output_filter(request, &out); } } current_buf_pos += chunk_len; /* TODO: More Codes to Catch? */ if (rc == NGX_ERROR) { gridfile_destroy(&gfile); gridfs_destroy(&gfs); return NGX_ERROR; } } gridfile_destroy(&gfile); gridfs_destroy(&gfs); return rc; }
static ngx_int_t ngx_http_gridfs_handler(ngx_http_request_t* request) { ngx_http_gridfs_loc_conf_t* gridfs_conf; ngx_http_core_loc_conf_t* core_conf; ngx_buf_t* buffer; ngx_chain_t out; ngx_str_t location_name; ngx_str_t full_uri; // --------------- xulin add start ------------------- char* ml_args; char* arg; unsigned int add_arg; unsigned int add_len; // --------------- xulin add end ------------------- char* value; ngx_http_mongo_connection_t *mongo_conn; gridfs gfs; gridfile gfile; gridfs_offset length; ngx_uint_t numchunks; char* contenttype; char* md5; bson_date_t last_modified; volatile ngx_uint_t i; ngx_int_t rc = NGX_OK; bson query; bson_oid_t oid; mongo_cursor ** cursors; gridfs_offset chunk_len; const char * chunk_data; bson_iterator it; bson chunk; ngx_pool_cleanup_t* gridfs_cln; ngx_http_gridfs_cleanup_t* gridfs_clndata; int status; volatile ngx_uint_t e = FALSE; volatile ngx_uint_t ecounter = 0; gridfs_conf = ngx_http_get_module_loc_conf(request, ngx_http_gridfs_module); core_conf = ngx_http_get_module_loc_conf(request, ngx_http_core_module); // ---------- ENSURE MONGO CONNECTION ---------- // mongo_conn = ngx_http_get_mongo_connection( gridfs_conf->mongo ); if (mongo_conn == NULL) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Mongo Connection not found: \"%V\"", &gridfs_conf->mongo); return NGX_HTTP_INTERNAL_SERVER_ERROR; } if ( !(&mongo_conn->conn.connected) && (ngx_http_mongo_reconnect(request->connection->log, mongo_conn) == NGX_ERROR || ngx_http_mongo_reauth(request->connection->log, mongo_conn) == NGX_ERROR)) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Could not connect to mongo: \"%V\"", &gridfs_conf->mongo); if(&mongo_conn->conn.connected) { mongo_disconnect(&mongo_conn->conn); } return NGX_HTTP_SERVICE_UNAVAILABLE; } // ---------- RETRIEVE KEY ---------- // location_name = core_conf->name; full_uri = request->uri; if (full_uri.len < location_name.len) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Invalid location name or uri."); return NGX_HTTP_INTERNAL_SERVER_ERROR; } value = (char*)malloc(sizeof(char) * (full_uri.len - location_name.len + 1 + request->args.len + 1)); if (value == NULL) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Failed to allocate memory for value buffer."); return NGX_HTTP_INTERNAL_SERVER_ERROR; } memcpy(value, full_uri.data + location_name.len, full_uri.len - location_name.len); // ------------------------------------ xulin add start -------------------------------------- if (request->args.len > 0) { ml_args = (char*)malloc(sizeof(char) * (request->args.len + 1)); memcpy(ml_args, request->args.data, request->args.len); ml_args[request->args.len] = '\0'; add_len = full_uri.len - location_name.len; memcpy(value + add_len, "?", 1); add_len += 1; arg = strtok(ml_args, "&"); while (arg != NULL) { add_arg = 1; if (strstr(arg, "xc_md5") != NULL) { add_arg = 0; } else if (strstr(arg, "_xingcloud_t") != NULL) { add_arg = 0; } if (add_arg == 1) { memcpy(value + add_len, arg, strlen(arg)); add_len += strlen(arg); memcpy(value + add_len, "&", 1); add_len += 1; } arg = strtok(NULL, "&"); } free(ml_args); if (value[add_len - 1] == '?' || value[add_len - 1] == '&') { value[add_len - 1] = '\0'; } else { value[add_len] = '\0'; } } ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "ml_url = [%s]", value); // ------------------------------------ xulin add end -------------------------------------- if (!url_decode(value)) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Malformed request."); free(value); return NGX_HTTP_BAD_REQUEST; } // ---------- RETRIEVE GRIDFILE ---------- // do { e = FALSE; if (gridfs_init(&mongo_conn->conn, (const char*)gridfs_conf->db.data, (const char*)gridfs_conf->root_collection.data, &gfs) != MONGO_OK) { e = TRUE; ecounter++; if (ecounter > MONGO_MAX_RETRIES_PER_REQUEST || ngx_http_mongo_reconnect(request->connection->log, mongo_conn) == NGX_ERROR || ngx_http_mongo_reauth(request->connection->log, mongo_conn) == NGX_ERROR) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Mongo connection dropped, could not reconnect"); if(&mongo_conn->conn.connected) { mongo_disconnect(&mongo_conn->conn); } free(value); return NGX_HTTP_SERVICE_UNAVAILABLE; } } } while (e); bson_init(&query); switch (gridfs_conf->type) { case BSON_OID: bson_oid_from_string(&oid, value); bson_append_oid(&query, (char*)gridfs_conf->field.data, &oid); break; case BSON_INT: bson_append_int(&query, (char*)gridfs_conf->field.data, ngx_atoi((u_char*)value, strlen(value))); break; case BSON_STRING: bson_append_string(&query, (char*)gridfs_conf->field.data, value); break; } bson_finish(&query); status = gridfs_find_query(&gfs, &query, &gfile); bson_destroy(&query); free(value); if(status == MONGO_ERROR) { gridfs_destroy(&gfs); return NGX_HTTP_NOT_FOUND; } /* Get information about the file */ length = gridfile_get_contentlength(&gfile); numchunks = gridfile_get_numchunks(&gfile); contenttype = (char*)gridfile_get_contenttype(&gfile); md5 = (char*)gridfile_get_md5(&gfile); last_modified = gridfile_get_uploaddate(&gfile); // ---------- SEND THE HEADERS ---------- // request->headers_out.status = NGX_HTTP_OK; request->headers_out.content_length_n = length; if (contenttype != NULL) { request->headers_out.content_type.len = strlen(contenttype); request->headers_out.content_type.data = (u_char*)contenttype; } else ngx_http_set_content_type(request); // use md5 field as ETag if possible if (md5 != NULL) { request->headers_out.etag = ngx_list_push(&request->headers_out.headers); request->headers_out.etag->hash = 1; request->headers_out.etag->key.len = sizeof("ETag") - 1; request->headers_out.etag->key.data = (u_char*)"ETag"; ngx_buf_t *b; b = ngx_create_temp_buf(request->pool, strlen(md5) + 2); b->last = ngx_sprintf(b->last, "\"%s\"", md5); request->headers_out.etag->value.len = strlen(md5) + 2; request->headers_out.etag->value.data = b->start; } // use uploadDate field as last_modified if possible if (last_modified) { request->headers_out.last_modified_time = (time_t)(last_modified/1000); } /* Determine if content is gzipped, set headers accordingly */ if ( gridfile_get_boolean(&gfile,"gzipped") ) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, gridfile_get_field(&gfile,"gzipped") ); request->headers_out.content_encoding = ngx_list_push(&request->headers_out.headers); if (request->headers_out.content_encoding == NULL) { gridfile_destroy(&gfile); gridfs_destroy(&gfs); return NGX_ERROR; } request->headers_out.content_encoding->hash = 1; request->headers_out.content_encoding->key.len = sizeof("Content-Encoding") - 1; request->headers_out.content_encoding->key.data = (u_char *) "Content-Encoding"; request->headers_out.content_encoding->value.len = sizeof("gzip") - 1; request->headers_out.content_encoding->value.data = (u_char *) "gzip"; } ngx_http_send_header(request); // ---------- SEND THE BODY ---------- // /* Empty file */ if (numchunks == 0) { /* Allocate space for the response buffer */ buffer = ngx_pcalloc(request->pool, sizeof(ngx_buf_t)); if (buffer == NULL) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Failed to allocate response buffer"); gridfile_destroy(&gfile); gridfs_destroy(&gfs); return NGX_HTTP_INTERNAL_SERVER_ERROR; } buffer->pos = NULL; buffer->last = NULL; buffer->memory = 1; buffer->last_buf = 1; out.buf = buffer; out.next = NULL; gridfile_destroy(&gfile); gridfs_destroy(&gfs); return ngx_http_output_filter(request, &out); } cursors = (mongo_cursor **)ngx_pcalloc(request->pool, sizeof(mongo_cursor *) * numchunks); if (cursors == NULL) { gridfile_destroy(&gfile); gridfs_destroy(&gfs); return NGX_HTTP_INTERNAL_SERVER_ERROR; } ngx_memzero( cursors, sizeof(mongo_cursor *) * numchunks); /* Hook in the cleanup function */ gridfs_cln = ngx_pool_cleanup_add(request->pool, sizeof(ngx_http_gridfs_cleanup_t)); if (gridfs_cln == NULL) { gridfile_destroy(&gfile); gridfs_destroy(&gfs); return NGX_HTTP_INTERNAL_SERVER_ERROR; } gridfs_cln->handler = ngx_http_gridfs_cleanup; gridfs_clndata = gridfs_cln->data; gridfs_clndata->cursors = cursors; gridfs_clndata->numchunks = numchunks; /* Read and serve chunk by chunk */ for (i = 0; i < numchunks; i++) { /* Allocate space for the response buffer */ buffer = ngx_pcalloc(request->pool, sizeof(ngx_buf_t)); if (buffer == NULL) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Failed to allocate response buffer"); gridfile_destroy(&gfile); gridfs_destroy(&gfs); return NGX_HTTP_INTERNAL_SERVER_ERROR; } /* Fetch the chunk from mongo */ do { e = FALSE; cursors[i] = gridfile_get_chunks(&gfile, i, 1); if (!(cursors[i] && mongo_cursor_next(cursors[i]) == MONGO_OK)) { e = TRUE; ecounter++; if (ecounter > MONGO_MAX_RETRIES_PER_REQUEST || ngx_http_mongo_reconnect(request->connection->log, mongo_conn) == NGX_ERROR || ngx_http_mongo_reauth(request->connection->log, mongo_conn) == NGX_ERROR) { ngx_log_error(NGX_LOG_ERR, request->connection->log, 0, "Mongo connection dropped, could not reconnect"); if(&mongo_conn->conn.connected) { mongo_disconnect(&mongo_conn->conn); } gridfile_destroy(&gfile); gridfs_destroy(&gfs); return NGX_HTTP_SERVICE_UNAVAILABLE; } } } while (e); chunk = cursors[i]->current; bson_find(&it, &chunk, "data"); chunk_len = bson_iterator_bin_len( &it ); chunk_data = bson_iterator_bin_data( &it ); /* Set up the buffer chain */ buffer->pos = (u_char*)chunk_data; buffer->last = (u_char*)chunk_data + chunk_len; buffer->memory = 1; buffer->last_buf = (i == numchunks-1); out.buf = buffer; out.next = NULL; /* Serve the Chunk */ rc = ngx_http_output_filter(request, &out); /* TODO: More Codes to Catch? */ if (rc == NGX_ERROR) { gridfile_destroy(&gfile); gridfs_destroy(&gfs); return NGX_ERROR; } } gridfile_destroy(&gfile); gridfs_destroy(&gfs); return rc; }
static void test_gridfile( gridfs *gfs, char *data_before, int64_t length, char *filename, char *content_type ) { gridfile gfile[1]; FILE *stream; #ifdef DYING mongo_md5_state_t pms[1]; mongo_md5_byte_t digest[16]; #endif char hex_digest[33]; int64_t i = length; int n; char *data_after = (char*)bson_malloc( LARGE ); int truncBytes; char* lowerName; ASSERT(gridfs_find_filename( gfs, filename, gfile ) == MONGO_OK); ASSERT( gridfile_exists( gfile ) ); stream = fopen( "output", "w+" ); gridfile_write_file( gfile, stream ); fseek( stream, 0, SEEK_SET ); ASSERT( fread( data_after, (size_t)length, sizeof( char ), stream ) ); fclose( stream ); ASSERT( memcmp( data_before, data_after, (size_t)length ) == 0 ); gridfile_read_buffer( gfile, data_after, length ); ASSERT( memcmp( data_before, data_after, (size_t)length ) == 0 ); lowerName = (char*) bson_malloc( (int)strlen( filename ) + 1); strcpy( lowerName, filename ); _strlwr( lowerName ); ASSERT( strcmp( gridfile_get_filename( gfile ), lowerName ) == 0 ); bson_free( lowerName ); ASSERT( gridfile_get_contentlength( gfile ) == (size_t)length ); ASSERT( gridfile_get_chunksize( gfile ) == DEFAULT_CHUNK_SIZE ); ASSERT( strcmp( gridfile_get_contenttype( gfile ), content_type ) == 0 ) ; ASSERT( memcmp( data_before, data_after, (size_t)length ) == 0 ); if( !( gfile->flags & GRIDFILE_COMPRESS ) ) { #ifdef DYING mongo_md5_init( pms ); n = 0; while( i > INT_MAX ) { mongo_md5_append( pms, ( const mongo_md5_byte_t * )data_before + ( n * INT_MAX ), INT_MAX ); i -= INT_MAX; n += 1; } if( i > 0 ) mongo_md5_append( pms, ( const mongo_md5_byte_t * )data_before + ( n * INT_MAX ), (int)i ); mongo_md5_finish( pms, digest ); digest2hex( digest, hex_digest ); #else { DIGEST_CTX ctx = rpmDigestInit(PGPHASHALGO_MD5, RPMDIGEST_NONE); const char * _digest = NULL; int xx; while( i > INT_MAX ) { xx = rpmDigestUpdate(ctx, (char *)data_before + (n*INT_MAX), INT_MAX); i -= INT_MAX; n += 1; } xx = rpmDigestFinal(ctx, &_digest, NULL, 1); strncpy(hex_digest, _digest, 32+1); hex_digest[32] = '\0'; _digest = _free(_digest); } #endif ASSERT( strcmp( gridfile_get_md5( gfile ), hex_digest ) == 0 ); } truncBytes = (int) (length > DEFAULT_CHUNK_SIZE * 4 ? length - DEFAULT_CHUNK_SIZE * 2 - 13 : 23); gridfile_writer_init( gfile, gfs, filename, content_type, GRIDFILE_DEFAULT); ASSERT( gridfile_truncate(gfile, (size_t)(length - truncBytes)) == (size_t)(length - truncBytes)); gridfile_writer_done( gfile ); gridfile_seek(gfile, 0); ASSERT( gridfile_get_contentlength( gfile ) == (size_t)(length - truncBytes) ); ASSERT( gridfile_read_buffer( gfile, data_after, length ) == (size_t)(length - truncBytes)); ASSERT( memcmp( data_before, data_after, (size_t)(length - truncBytes) ) == 0 ); gridfile_writer_init( gfile, gfs, filename, content_type, GRIDFILE_DEFAULT); gridfile_truncate(gfile, 0); gridfile_writer_done( gfile ); ASSERT( gridfile_get_contentlength( gfile ) == 0 ); ASSERT( gridfile_read_buffer( gfile, data_after, length ) == 0 ); gridfile_destroy( gfile ); ASSERT( gridfs_remove_filename( gfs, filename ) == MONGO_OK ); free( data_after ); gridfs_test_unlink( "output" ); }
static void test_large( void ) { mongo conn[1]; gridfs gfs[1]; gridfile gfile[1]; FILE *fd; size_t i, n; char *buffer = (char*)bson_malloc( LARGE ); char *read_buf = (char*)bson_malloc( LARGE ); gridfs_offset filesize = ( int64_t )1024 * ( int64_t )LARGE; mongo_write_concern wc; bson lastError; bson lastErrorCmd; srand( (unsigned int) time( NULL ) ); INIT_SOCKETS_FOR_WINDOWS; CONN_CLIENT_TEST; mongo_write_concern_init(&wc); wc.j = 1; mongo_write_concern_finish(&wc); mongo_set_write_concern(conn, &wc); GFS_INIT; fd = fopen( "bigfile", "r" ); if( fd ) { fclose( fd ); } else { /* Create a very large file */ fill_buffer_randomly( buffer, ( int64_t )LARGE ); fd = fopen( "bigfile", "w" ); for( i=0; i<1024; i++ ) { fwrite( buffer, 1, LARGE, fd ); } fclose( fd ); } /* Now read the file into GridFS */ gridfs_remove_filename( gfs, "bigfile" ); gridfs_store_file( gfs, "bigfile", "bigfile", "text/html", GRIDFILE_NOMD5 | GRIDFILE_COMPRESS); gridfs_find_filename( gfs, "bigfile", gfile ); ASSERT( strcmp( gridfile_get_filename( gfile ), "bigfile" ) == 0 ); ASSERT( gridfile_get_contentlength( gfile ) == filesize ); fd = fopen( "bigfile", "r" ); while( ( n = fread( buffer, 1, MEDIUM, fd ) ) != 0 ) { ASSERT( gridfile_read_buffer( gfile, read_buf, MEDIUM ) == n ); ASSERT( memcmp( buffer, read_buf, n ) == 0 ); } fclose( fd ); gridfile_destroy( gfile ); /* Read the file using the streaming interface */ gridfs_remove_filename( gfs, "bigfile" ); gridfs_remove_filename( gfs, "bigfile-stream" ); gridfile_writer_init( gfile, gfs, "bigfile-stream", "text/html", GRIDFILE_NOMD5 | GRIDFILE_COMPRESS ); mongo_write_concern_destroy( &wc ); mongo_write_concern_init(&wc); wc.j = 0; /* Let's reset write concern j field to zero, we will manually call getLastError with j = 1 */ mongo_write_concern_finish(&wc); mongo_set_write_concern(conn, &wc); fd = fopen( "bigfile", "r" ); i = 0; while( ( n = fread( buffer, 1, READ_WRITE_BUF_SIZE, fd ) ) != 0 ) { ASSERT( gridfile_write_buffer( gfile, buffer, n ) == n ); if(i++ % 10 == 0) { bson_init( &lastErrorCmd ); bson_append_int( &lastErrorCmd, "getLastError", 1); bson_append_int( &lastErrorCmd, "j", 1); bson_finish( &lastErrorCmd ); bson_init( &lastError ); mongo_run_command( conn, "test", &lastErrorCmd, &lastError ); bson_destroy( &lastError ); bson_destroy( &lastErrorCmd ); } } mongo_write_concern_destroy( &wc ); mongo_write_concern_init(&wc); wc.j = 1; /* Let's reset write concern j field to 1 */ mongo_write_concern_finish(&wc); mongo_set_write_concern(conn, &wc); fclose( fd ); gridfile_writer_done( gfile ); gridfs_find_filename( gfs, "bigfile-stream", gfile ); ASSERT( strcmp( gridfile_get_filename( gfile ), "bigfile-stream" ) == 0 ); ASSERT( gridfile_get_contentlength( gfile ) == filesize ); gridfs_remove_filename( gfs, "bigfile-stream" ); gridfs_destroy( gfs ); mongo_disconnect( conn ); mongo_destroy( conn ); bson_free( buffer ); bson_free( read_buf ); mongo_write_concern_destroy( &wc ); }
EXPORT void mongo_gridfile_destroy(struct gridfile_* gf) { if (gf) { gridfile_destroy((gridfile*)gf); free(gf); } }
static void mongoGridfileFinalizer(SEXP ptr) { if (!R_ExternalPtrAddr(ptr)) return; gridfile_destroy((gridfile*)R_ExternalPtrAddr(ptr)); R_ClearExternalPtr(ptr); }
SEXP mongo_gridfile_destroy(SEXP gfile) { gridfile* _gfile = _checkGridfile(gfile); gridfile_destroy(_gfile); R_ClearExternalPtr(getAttrib(gfile, sym_mongo_gridfile)); return R_NilValue; }
void test_gridfile( gridfs *gfs, char *data_before, int64_t length, char *filename, char *content_type ) { gridfile gfile[1]; FILE *stream; mongo_md5_state_t pms[1]; mongo_md5_byte_t digest[16]; char hex_digest[33]; int64_t i = length; int n; char *data_after = (char*)bson_malloc( LARGE ); int truncBytes; char* lowerName; ASSERT(gridfs_find_filename( gfs, filename, gfile ) == MONGO_OK); ASSERT( gridfile_exists( gfile ) ); stream = fopen( "output", "w+" ); gridfile_write_file( gfile, stream ); fseek( stream, 0, SEEK_SET ); ASSERT( fread( data_after, (size_t)length, sizeof( char ), stream ) ); fclose( stream ); ASSERT( memcmp( data_before, data_after, (size_t)length ) == 0 ); gridfile_read( gfile, length, data_after ); ASSERT( memcmp( data_before, data_after, (size_t)length ) == 0 ); lowerName = (char*) bson_malloc( (int)strlen( filename ) + 1); strcpy( lowerName, filename); _strlwr( lowerName ); ASSERT( strcmp( gridfile_get_filename( gfile ), lowerName ) == 0 ); bson_free( lowerName ); ASSERT( gridfile_get_contentlength( gfile ) == (size_t)length ); ASSERT( gridfile_get_chunksize( gfile ) == DEFAULT_CHUNK_SIZE ); ASSERT( strcmp( gridfile_get_contenttype( gfile ), content_type ) == 0 ) ; ASSERT( memcmp( data_before, data_after, (size_t)length ) == 0 ); if( !( gfile->flags & GRIDFILE_COMPRESS ) ) { mongo_md5_init( pms ); n = 0; while( i > INT_MAX ) { mongo_md5_append( pms, ( const mongo_md5_byte_t * )data_before + ( n * INT_MAX ), INT_MAX ); i -= INT_MAX; n += 1; } if( i > 0 ) mongo_md5_append( pms, ( const mongo_md5_byte_t * )data_before + ( n * INT_MAX ), (int)i ); mongo_md5_finish( pms, digest ); digest2hex( digest, hex_digest ); ASSERT( strcmp( gridfile_get_md5( gfile ), hex_digest ) == 0 ); } truncBytes = (int) (length > DEFAULT_CHUNK_SIZE * 4 ? length - DEFAULT_CHUNK_SIZE * 2 - 13 : 23); gridfile_writer_init( gfile, gfs, filename, content_type, GRIDFILE_DEFAULT); ASSERT( gridfile_truncate(gfile, (size_t)(length - truncBytes)) == (size_t)(length - truncBytes)); gridfile_writer_done( gfile ); gridfile_seek(gfile, 0); ASSERT( gridfile_get_contentlength( gfile ) == (size_t)(length - truncBytes) ); ASSERT( gridfile_read( gfile, length, data_after ) == (size_t)(length - truncBytes)); ASSERT( memcmp( data_before, data_after, (size_t)(length - truncBytes) ) == 0 ); gridfile_writer_init( gfile, gfs, filename, content_type, GRIDFILE_DEFAULT); gridfile_truncate(gfile, 0); gridfile_writer_done( gfile ); ASSERT( gridfile_get_contentlength( gfile ) == 0 ); ASSERT( gridfile_read( gfile, length, data_after ) == 0 ); gridfile_destroy( gfile ); gridfs_remove_filename( gfs, filename ); free( data_after ); _unlink( "output" ); }