int gridfs_open(const char *path, struct fuse_file_info *fi)
{
path = fuse_to_mongo_path(path);
if((fi->flags & O_ACCMODE) == O_RDONLY) {
if(open_files.find(path) != open_files.end()) {
return 0;
}
ScopedDbConnection sdc(*gridfs_options.conn_string);
ScopedDbConnection_init(sdc);
GridFS gf(sdc.conn(), gridfs_options.db, gridfs_options.prefix);
GridFile file = gf.findFile(path);
sdc.done();
if(file.exists()) {
return 0;
}
return -ENOENT;
} else {
return -EACCES;
}
}
bool LocalMemoryGridFile::initLocalBuffers(GridFile& gridFile) {
if (!setSize(gridFile.getContentLength())) {
return false;
}
//TODO: Check for size compared to the content length that was specified for consistency
off_t offset = 0;
int chunkCount = gridFile.getNumChunks();
for (int i = 0; i < chunkCount; ++i) {
//TODO: Error checking or getChunk and related handling
GridFSChunk chunk = gridFile.getChunk(i);
int chunkLen = 0;
const char* data = chunk.data(chunkLen);
if (!data) {
error() << "Failed to get data from expected chunk {file: " << _filename
<< ", chunkOffset: " << i << ", totalChunks: " << chunkCount
<< "}, will stop in-between and return error." << endl;
return false;
}
_write(data, chunkLen, offset);
offset += chunkLen;
}
return true;
}
/*
* bool = gridfile:exists()
*/
static int gridfile_exists(lua_State *L) {
GridFile *gridfile = userdata_to_gridfile(L, 1);
lua_pushboolean(L, gridfile->exists());
return 1;
}
/*
* md5_str = gridfile:md5()
*/
static int gridfile_md5(lua_State *L) {
GridFile *gridfile = userdata_to_gridfile(L, 1);
lua_pushstring(L, gridfile->getMD5().c_str());
return 1;
}
/*
* num_chunks = gridfile:num_chunks()
*/
static int gridfile_num_chunks(lua_State *L) {
GridFile *gridfile = userdata_to_gridfile(L, 1);
lua_pushinteger(L, gridfile->getNumChunks());
return 1;
}
/*
* chunk_size = gridfile:chunk_size()
*/
static int gridfile_chunk_size(lua_State *L) {
GridFile *gridfile = userdata_to_gridfile(L, 1);
lua_pushinteger(L, gridfile->getChunkSize());
return 1;
}
int gridfs_open(const char *path, struct fuse_file_info *fi)
{
path = fuse_to_mongo_path(path);
if((fi->flags & O_ACCMODE) == O_RDONLY) {
if(open_files.find(path) != open_files.end()) {
return 0;
}
ScopedDbConnection sdc(*gridfs_options.conn_string);
bool digest = true;
string err = "";
sdc.conn().DBClientWithCommands::auth(gridfs_options.db, gridfs_options.username, gridfs_options.password, err, digest);
fprintf(stderr, "DEBUG: %s\n", err.c_str());
GridFS gf(sdc.conn(), gridfs_options.db);
GridFile file = gf.findFile(path);
sdc.done();
if(file.exists()) {
return 0;
}
return -ENOENT;
} else {
return -EACCES;
}
}
/*
* content_length = gridfile:content_length()
* __len
*/
static int gridfile_content_length(lua_State *L) {
GridFile *gridfile = userdata_to_gridfile(L, 1);
lua_pushnumber(L, gridfile->getContentLength());
return 1;
}
/*
* metadata_table = gridfile:metadata()
*/
static int gridfile_metadata(lua_State *L) {
GridFile *gridfile = userdata_to_gridfile(L, 1);
bson_to_lua(L, gridfile->getMetadata());
return 1;
}
int gridfs_getattr(const char *path, struct stat *stbuf)
{
int res = 0;
memset(stbuf, 0, sizeof(struct stat));
if(strcmp(path, "/") == 0) {
stbuf->st_mode = S_IFDIR | 0777;
stbuf->st_nlink = 2;
return 0;
}
path = fuse_to_mongo_path(path);
map<string,LocalGridFile*>::const_iterator file_iter;
file_iter = open_files.find(path);
if(file_iter != open_files.end()) {
stbuf->st_mode = S_IFREG | 0555;
stbuf->st_nlink = 1;
stbuf->st_ctime = time(NULL);
stbuf->st_mtime = time(NULL);
stbuf->st_size = file_iter->second->getLength();
return 0;
}
// HACK: This is a protective measure to ensure we don't spin off into forever if a path without a period is requested.
if(path_depth(path) > 10) {
return -ENOENT;
}
// HACK: Assumes that if the last part of the path has a '.' in it, it's the leaf of the path, and if we haven't found a match by now,
// give up and go home. This works just dandy as long as you avoid putting periods in your 'directory' names.
/*if(!is_leaf(path)) {
stbuf->st_mode = S_IFDIR | 0777;
stbuf->st_nlink = 2;
return 0;
}*/
ScopedDbConnection sdc(*gridfs_options.conn_string);
ScopedDbConnection_init(sdc);
GridFS gf(sdc.conn(), gridfs_options.db, gridfs_options.prefix);
GridFile file = gf.findFile(path);
sdc.done();
if(!file.exists()) {
return -ENOENT;
}
stbuf->st_mode = S_IFREG | 0555;
stbuf->st_nlink = 1;
stbuf->st_size = file.getContentLength();
time_t upload_time = mongo_time_to_unix_time(file.getUploadDate());
stbuf->st_ctime = upload_time;
stbuf->st_mtime = upload_time;
return 0;
}
static int gridfile_field(lua_State *L) {
GridFile *gridfile = userdata_to_gridfile(L, 1);
const char *field_name = luaL_checkstring(L, 2);
BSONElement elem = gridfile->getFileField(field_name);
lua_push_value(L, elem);
return 1;
}
/*
* date = gridfile:upload_date()
*/
static int gridfile_upload_date(lua_State *L) {
GridFile *gridfile = userdata_to_gridfile(L, 1);
Date_t upload_date = gridfile->getUploadDate();
push_bsontype_table(L, mongo::Date);
lua_pushnumber(L, upload_date);
lua_rawseti(L, -2, 1);
return 1;
}
int gridfs_read(const char *path, char *buf, size_t size, off_t offset,
struct fuse_file_info *fi)
{
path = fuse_to_mongo_path(path);
size_t len = 0;
map<string,LocalGridFile*>::const_iterator file_iter;
file_iter = open_files.find(path);
if(file_iter != open_files.end()) {
LocalGridFile *lgf = file_iter->second;
return lgf->read(buf, size, offset);
}
ScopedDbConnection sdc(*gridfs_options.conn_string);
bool digest = true;
string err = "";
sdc.conn().DBClientWithCommands::auth(gridfs_options.db, gridfs_options.username, gridfs_options.password, err, digest);
fprintf(stderr, "DEBUG: %s\n", err.c_str());
GridFS gf(sdc.conn(), gridfs_options.db);
GridFile file = gf.findFile(path);
if(!file.exists()) {
sdc.done();
return -EBADF;
}
int chunk_size = file.getChunkSize();
int chunk_num = offset / chunk_size;
while(len < size && chunk_num < file.getNumChunks()) {
GridFSChunk chunk = file.getChunk(chunk_num);
int to_read;
int cl = chunk.len();
const char *d = chunk.data(cl);
if(len) {
to_read = min((long unsigned)cl, (long unsigned)(size - len));
memcpy(buf + len, d, to_read);
} else {
to_read = min((long unsigned)(cl - (offset % chunk_size)), (long unsigned)(size - len));
memcpy(buf + len, d + (offset % chunk_size), to_read);
}
len += to_read;
chunk_num++;
}
sdc.done();
return len;
}
int gridfs_getxattr(const char* path, const char* name, char* value, size_t size)
{
if(strcmp(path, "/") == 0) {
return -ENOATTR;
}
path = fuse_to_mongo_path(path);
const char* attr_name = unnamespace_xattr(name);
if(!attr_name) {
return -ENOATTR;
}
if(open_files.find(path) != open_files.end()) {
return -ENOATTR;
}
ScopedDbConnection sdc(*gridfs_options.conn_string);
bool digest = true;
string err = "";
sdc.conn().DBClientWithCommands::auth(gridfs_options.db, gridfs_options.username, gridfs_options.password, err, digest);
fprintf(stderr, "DEBUG: %s\n", err.c_str());
GridFS gf(sdc.conn(), gridfs_options.db);
GridFile file = gf.findFile(path);
sdc.done();
if(!file.exists()) {
return -ENOENT;
}
BSONObj metadata = file.getMetadata();
if(metadata.isEmpty()) {
return -ENOATTR;
}
BSONElement field = metadata[attr_name];
if(field.eoo()) {
return -ENOATTR;
}
string field_str = field.toString();
int len = field_str.size() + 1;
if(size == 0) {
return len;
} else if(size < len) {
return -ERANGE;
}
memcpy(value, field_str.c_str(), len);
return len;
}
int gridfs_getxattr(const char* path, const char* name, char* value, size_t size)
#endif /* __APPLE__ */
{
if(strcmp(path, "/") == 0) {
return -ENOATTR;
}
path = fuse_to_mongo_path(path);
const char* attr_name = unnamespace_xattr(name);
if(!attr_name) {
return -ENOATTR;
}
if(open_files.find(path) != open_files.end()) {
return -ENOATTR;
}
ScopedDbConnection sdc(*gridfs_options.conn_string);
ScopedDbConnection_init(sdc);
GridFS gf(sdc.conn(), gridfs_options.db, gridfs_options.prefix);
GridFile file = gf.findFile(path);
sdc.done();
if(!file.exists()) {
return -ENOENT;
}
BSONObj metadata = file.getMetadata();
if(metadata.isEmpty()) {
return -ENOATTR;
}
BSONElement field = metadata[attr_name];
if(field.eoo()) {
return -ENOATTR;
}
string field_str = field.toString();
int len = field_str.size() + 1;
if(size == 0) {
return len;
} else if(size < len) {
return -ERANGE;
}
memcpy(value, field_str.c_str(), len);
return len;
}
/*
* success,err = gridfile:write(filename)
*/
static int gridfile_write(lua_State *L) {
GridFile *gridfile = userdata_to_gridfile(L, 1);
const char *where = luaL_checkstring(L, 2);
try {
gridfile->write(lua_tostring(L, 2));
} catch (std::exception &e) {
lua_pushboolean(L, 0);
lua_pushfstring(L, LUAMONGO_ERR_CALLING, LUAMONGO_GRIDFILE, "write", e.what());
return 2;
}
lua_pushboolean(L, 1);
return 1;
}
int gridfs_read(const char *path, char *buf, size_t size, off_t offset,
struct fuse_file_info *fi)
{
path = fuse_to_mongo_path(path);
size_t len = 0;
map<string,LocalGridFile*>::const_iterator file_iter;
file_iter = open_files.find(path);
if(file_iter != open_files.end()) {
LocalGridFile *lgf = file_iter->second;
return lgf->read(buf, size, offset);
}
ScopedDbConnection sdc(*gridfs_options.conn_string);
ScopedDbConnection_init(sdc);
GridFS gf(sdc.conn(), gridfs_options.db, gridfs_options.prefix);
GridFile file = gf.findFile(path);
if(!file.exists()) {
sdc.done();
return -EBADF;
}
int chunk_size = file.getChunkSize();
int chunk_num = offset / chunk_size;
while(len < size && chunk_num < file.getNumChunks()) {
GridFSChunk chunk = file.getChunk(chunk_num);
int to_read;
int cl = chunk.len();
const char *d = chunk.data(cl);
if(len) {
to_read = min((long unsigned)cl, (long unsigned)(size - len));
memcpy(buf + len, d, to_read);
} else {
to_read = min((long unsigned)(cl - (offset % chunk_size)), (long unsigned)(size - len));
memcpy(buf + len, d + (offset % chunk_size), to_read);
}
len += to_read;
chunk_num++;
}
sdc.done();
return len;
}
/*
* string = gridfile:data()
*/
static int gridfile_data(lua_State *L) {
GridFile *gridfile = userdata_to_gridfile(L, 1);
std::stringstream data(std::stringstream::out | std::stringstream::binary);
try {
gridfile->write(data);
} catch (std::exception &e) {
lua_pushboolean(L, 0);
lua_pushfstring(L, LUAMONGO_ERR_CALLING, LUAMONGO_GRIDFILE, "data", e.what());
return 2;
}
std::string data_str = data.str();
lua_pushlstring (L, data_str.c_str(), data_str.length());
return 1;
}
void OutCoreInterp::updateGridPoint(int fileNum, int x, int y, double data_z, double distance)
{
unsigned int coord = y * local_grid_size_x + x;
GridFile *gf = gridMap[fileNum]->getGridFile();
if(gf == NULL || gf->interp == NULL)
{
//cout << "OutCoreInterp::updateGridPoint() gridFile is NULL" << endl;
return;
}
if(coord < gf->getMemSize() && coord != 0)
{
if(gf->interp[coord].Zmin > data_z)
gf->interp[coord].Zmin = data_z;
if(gf->interp[coord].Zmax < data_z)
gf->interp[coord].Zmax = data_z;
gf->interp[coord].Zmean += data_z;
gf->interp[coord].count++;
/*
// same as InCoreInterp::updateGridPoint
double delta = data_z - gf->interp[coord].Zstd_tmp;
gf->interp[coord].Zstd_tmp += delta/gf->interp[coord].count;
gf->interp[coord].Zstd += delta * (data_z - gf->interp[coord].Zstd_tmp);
*/
double dist = pow(distance, Interpolation::WEIGHTER);
if (gf->interp[coord].sum != -1) {
if (dist != 0) {
gf->interp[coord].Zidw += data_z/dist;
gf->interp[coord].sum += 1/dist;
} else {
gf->interp[coord].Zidw = data_z;
gf->interp[coord].sum = -1;
}
} else {
// do nothing
}
} else {
cerr << "OutCoreInterp::updateGridPoint() Memory Access Violation! " << endl;
}
}
int gridfs_listxattr(const char* path, char* list, size_t size)
{
path = fuse_to_mongo_path(path);
if(open_files.find(path) != open_files.end()) {
return 0;
}
ScopedDbConnection sdc(*gridfs_options.conn_string);
bool digest = true;
string err = "";
sdc.conn().DBClientWithCommands::auth(gridfs_options.db, gridfs_options.username, gridfs_options.password, err, digest);
fprintf(stderr, "DEBUG: %s\n", err.c_str());
GridFS gf(sdc.conn(), gridfs_options.db);
GridFile file = gf.findFile(path);
sdc.done();
if(!file.exists()) {
return -ENOENT;
}
int len = 0;
BSONObj metadata = file.getMetadata();
set<string> field_set;
metadata.getFieldNames(field_set);
for(set<string>::const_iterator s = field_set.begin(); s != field_set.end(); s++) {
string attr_name = namespace_xattr(*s);
int field_len = attr_name.size() + 1;
len += field_len;
if(size >= len) {
memcpy(list, attr_name.c_str(), field_len);
list += field_len;
}
}
if(size == 0) {
return len;
} else if(size < len) {
return -ERANGE;
}
return len;
}
OutCoreInterp::~OutCoreInterp()
{
/*
if(qlist != NULL)
delete qlist;
*/
for(int i = 0; i < numFiles; i++)
{
//cout << "gridMap " << i << " deleted" << endl;
GridFile* f = gridMap[i]->getGridFile();
// Try to wipe the Grid data
int status = remove (f->getFileName().c_str());
if (status != 0)
std::cerr << "unable to remove tmpfile '" << f->getFileName() << "'" << std::endl;
delete gridMap[i];
}
if(gridMap != NULL)
delete [] gridMap;
}
int gridfs_listxattr(const char* path, char* list, size_t size)
{
path = fuse_to_mongo_path(path);
if(open_files.find(path) != open_files.end()) {
return 0;
}
ScopedDbConnection sdc(*gridfs_options.conn_string);
ScopedDbConnection_init(sdc);
GridFS gf(sdc.conn(), gridfs_options.db, gridfs_options.prefix);
GridFile file = gf.findFile(path);
sdc.done();
if(!file.exists()) {
return -ENOENT;
}
int len = 0;
BSONObj metadata = file.getMetadata();
set<string> field_set;
metadata.getFieldNames(field_set);
for(set<string>::const_iterator s = field_set.begin(); s != field_set.end(); s++) {
string attr_name = namespace_xattr(*s);
int field_len = attr_name.size() + 1;
len += field_len;
if(size >= len) {
memcpy(list, attr_name.c_str(), field_len);
list += field_len;
}
}
if(size == 0) {
return len;
} else if(size < len) {
return -ERANGE;
}
return len;
}
/*
* chunk, err = gridfile:chunk(chunk_num)
*/
static int gridfile_chunk(lua_State *L) {
GridFile *gridfile = userdata_to_gridfile(L, 1);
int num = luaL_checkint(L, 2);
int resultcount = 1;
try {
CHUNK c = gridfile->getChunk(num);
CHUNK *chunk_ptr = new CHUNK(c);
CHUNK **chunk = (CHUNK **)lua_newuserdata(L, sizeof(CHUNK *));
*chunk = chunk_ptr;
luaL_getmetatable(L, LUAMONGO_GRIDFSCHUNK);
lua_setmetatable(L, -2);
} catch (std::exception &e) {
lua_pushnil(L);
lua_pushfstring(L, LUAMONGO_ERR_GRIDFSCHUNK_FAILED, e.what());
resultcount = 2;
}
return resultcount;
}
BOOL CMongoDB::GetFileFromMongoDB(const string& strFileName, const string&strSavePath, string& strErr)
{
BOOL bRet = TRUE;
string strTemp, strTempPath;
int nStart(0);
if (strFileName.length() <= 0)
strErr = "FileName is empty!";
gridfs_offset size(0), afsize(0);
string strPath(strSavePath);
if (connect.isStillConnected() && m_bConnect)
{
GridFS fs(connect, m_strDBName);
auto pos = strFileName.find(";", nStart);
while ((pos > 0) && (pos != strFileName.npos))
{
strTemp = strFileName.substr(nStart, pos - nStart);
nStart = pos + 1;
if (strTemp.length() > 0)
{
GridFile file = fs.findFileByName(strTemp);
if (file.exists())
{
strPath = strSavePath;
strPath.append("\\");
strPath.append(strTemp);
size = file.getContentLength();
afsize = file.write(strPath);
}
else strErr = "File not exists!";
}
if (size > 0 && size == afsize)
bRet = bRet ? TRUE : FALSE;
else bRet = FALSE;
pos = strFileName.find(";", nStart);
}
}
return bRet;
}
int LocalMemoryGridFile::openRemote(int fileFlags) {
trace() << " -> LocalMemoryGridFile::openRemote {fileFlags: " << fileFlags << "}" << endl;
try {
ScopedDbConnection dbc(globalFSOptions._connectString);
GridFS gridFS(dbc.conn(), globalFSOptions._db, globalFSOptions._collPrefix);
GridFile origGridFile = gridFS.findFile(BSON("filename" << _filename));
if (!origGridFile.exists()) {
dbc.done();
error() << "Requested file not found for opening from remote {file: " << _filename << "}" << endl;
return -EBADF;
}
if (origGridFile.getContentLength() > globalFSOptions._maxMemFileSize) {
// Don't support opening files of size > MAX_MEMORY_FILE_CAPACITY in R/W mode
error() << "Requested file length is beyond supported length for in-memory files {file: "
<< _filename << ", length: {requested: " << origGridFile.getContentLength()
<< ", max-supported: " << globalFSOptions._maxMemFileSize << "} }" << endl;
return -EROFS;
}
if (!initLocalBuffers(origGridFile)) {
error() << "Failed to initialize local buffers {file: " << _filename << "}" << endl;
return -EIO;
}
_dirty = false;
dbc.done();
} catch (DBException& e) {
// Something failed in getting the file from GridFS
error() << "Caught exception in getting remote file for flush {filename: " << _filename
<< ", code: " << e.getCode() << ", what: " << e.what() << ", exception: " << e.toString()
<< "}" << endl;
return -EIO;
}
return 0;
}
/*
* gridfile, err = gridfs:find_file(filename)
*/
static int gridfs_find_file(lua_State *L) {
GridFS *gridfs = userdata_to_gridfs(L, 1);
int resultcount = 1;
if (!lua_isnoneornil(L, 2)) {
try {
int type = lua_type(L, 2);
if (type == LUA_TTABLE) {
BSONObj obj;
lua_to_bson(L, 2, obj);
GridFile gridfile = gridfs->findFile(obj);
if(gridfile.exists()) {
resultcount = gridfile_create(L, gridfile);
} else {
lua_pushnil(L);
resultcount = 1;
}
} else {
GridFile gridfile = gridfs->findFile(luaL_checkstring(L, 2));
if(gridfile.exists()) {
resultcount = gridfile_create(L, gridfile);
} else {
lua_pushnil(L);
resultcount = 1;
}
}
} catch (std::exception &e) {
lua_pushboolean(L, 0);
lua_pushfstring(L, LUAMONGO_ERR_CALLING, LUAMONGO_GRIDFS, "find_file", e.what());
resultcount = 2;
}
}
return resultcount;
}
int OutCoreInterp::outputFile(char *outputName, int outputFormat, unsigned int outputType)
{
int i, j, k, l;
FILE **arcFiles;
char arcFileName[1024];
FILE **gridFiles;
char gridFileName[1024];
char *ext[5] = {".min", ".max", ".mean", ".idw", ".den"};
unsigned int type[5] = {OUTPUT_TYPE_MIN, OUTPUT_TYPE_MAX, OUTPUT_TYPE_MEAN, OUTPUT_TYPE_IDW, OUTPUT_TYPE_DEN};
int numTypes = 5;
// open ArcGIS files
if(outputFormat == OUTPUT_FORMAT_ARC_ASCII || outputFormat == OUTPUT_FORMAT_ALL)
{
if((arcFiles = (FILE **)malloc(sizeof(FILE *) * numTypes)) == NULL)
{
cout << "Arc File open error: " << endl;
return -1;
}
for(i = 0; i < numTypes; i++)
{
if(outputType & type[i])
{
strncpy(arcFileName, outputName, sizeof(arcFileName));
strncat(arcFileName, ext[i], strlen(ext[i]));
strncat(arcFileName, ".asc", strlen(".asc"));
if((arcFiles[i] = fopen64(arcFileName, "w+")) == NULL)
{
cout << "File open error: " << arcFileName << endl;
return -1;
}
}else{
arcFiles[i] = NULL;
}
}
} else {
arcFiles = NULL;
}
// open Grid ASCII files
if(outputFormat == OUTPUT_FORMAT_GRID_ASCII || outputFormat == OUTPUT_FORMAT_ALL)
{
if((gridFiles = (FILE **)malloc(sizeof(FILE *) * numTypes)) == NULL)
{
cout << "File array allocation error" << endl;
return -1;
}
for(i = 0; i < numTypes; i++)
{
if(outputType & type[i])
{
strncpy(gridFileName, outputName, sizeof(arcFileName));
strncat(gridFileName, ext[i], strlen(ext[i]));
strncat(gridFileName, ".grid", strlen(".grid"));
if((gridFiles[i] = fopen64(gridFileName, "w+")) == NULL)
{
cout << "File open error: " << gridFileName << endl;
return -1;
}
}else{
gridFiles[i] = NULL;
}
}
} else {
gridFiles = NULL;
}
// print ArcGIS headers
if(arcFiles != NULL)
{
for(i = 0; i < numTypes; i++)
{
if(arcFiles[i] != NULL)
{
fprintf(arcFiles[i], "ncols %d\n", GRID_SIZE_X);
fprintf(arcFiles[i], "nrows %d\n", GRID_SIZE_Y);
fprintf(arcFiles[i], "xllcorner %f\n", min_x);
fprintf(arcFiles[i], "yllcorner %f\n", min_y);
fprintf(arcFiles[i], "cellsize %f\n", GRID_DIST_X);
fprintf(arcFiles[i], "NODATA_value -9999\n");
}
}
}
// print Grid headers
if(gridFiles != NULL)
{
for(i = 0; i < numTypes; i++)
{
if(gridFiles[i] != NULL)
{
fprintf(gridFiles[i], "north: %f\n", max_y);
fprintf(gridFiles[i], "south: %f\n", min_y);
fprintf(gridFiles[i], "east: %f\n", max_x);
fprintf(gridFiles[i], "west: %f\n", min_x);
fprintf(gridFiles[i], "rows: %d\n", GRID_SIZE_Y);
fprintf(gridFiles[i], "cols: %d\n", GRID_SIZE_X);
}
}
}
for(i = numFiles -1; i >= 0; i--)
{
GridFile *gf = gridMap[i]->getGridFile();
gf->map();
int start = gridMap[i]->getLowerBound() - gridMap[i]->getOverlapLowerBound();
int end = gridMap[i]->getUpperBound() - gridMap[i]->getOverlapLowerBound() + 1;
//int start = (gridMap[i]->getLowerBound() - gridMap[i]->getOverlapLowerBound()) * GRID_SIZE_X;
//int end = (gridMap[i]->getUpperBound() - gridMap[i]->getOverlapLowerBound() + 1) * GRID_SIZE_X;
cout << "Merging " << i << ": from " << (start) << " to " << (end) << endl;
cout << " " << i << ": from " << (start/GRID_SIZE_X) << " to " << (end/GRID_SIZE_X) << endl;
for(j = end - 1; j >= start; j--)
{
for(k = 0; k < GRID_SIZE_X; k++)
{
int index = j * GRID_SIZE_X + k;
if(arcFiles != NULL)
{
// Zmin
if(arcFiles[0] != NULL)
{
if(gf->interp[index].empty == 0 &&
gf->interp[index].filled == 0)
//if(gf->interp[k][j].Zmin == 0)
fprintf(arcFiles[0], "-9999 ");
else
//fprintf(arcFiles[0], "%f ", gf->interp[j][i].Zmin);
fprintf(arcFiles[0], "%f ", gf->interp[index].Zmin);
}
// Zmax
if(arcFiles[1] != NULL)
{
if(gf->interp[index].empty == 0 &&
gf->interp[index].filled == 0)
fprintf(arcFiles[1], "-9999 ");
else
fprintf(arcFiles[1], "%f ", gf->interp[index].Zmax);
}
// Zmean
if(arcFiles[2] != NULL)
{
if(gf->interp[index].empty == 0 &&
gf->interp[index].filled == 0)
fprintf(arcFiles[2], "-9999 ");
else
fprintf(arcFiles[2], "%f ", gf->interp[index].Zmean);
}
// Zidw
if(arcFiles[3] != NULL)
{
if(gf->interp[index].empty == 0 &&
gf->interp[index].filled == 0)
fprintf(arcFiles[3], "-9999 ");
else
fprintf(arcFiles[3], "%f ", gf->interp[index].Zidw);
}
// count
if(arcFiles[4] != NULL)
{
if(gf->interp[index].empty == 0 &&
gf->interp[index].filled == 0)
fprintf(arcFiles[4], "-9999 ");
else
fprintf(arcFiles[4], "%d ", gf->interp[index].count);
}
}
if(gridFiles != NULL)
{
// Zmin
if(gridFiles[0] != NULL)
{
if(gf->interp[index].empty == 0 &&
gf->interp[index].filled == 0)
fprintf(gridFiles[0], "-9999 ");
else
fprintf(gridFiles[0], "%f ", gf->interp[index].Zmin);
}
// Zmax
if(gridFiles[1] != NULL)
{
if(gf->interp[index].empty == 0 &&
gf->interp[index].filled == 0)
fprintf(gridFiles[1], "-9999 ");
else
fprintf(gridFiles[1], "%f ", gf->interp[index].Zmax);
}
// Zmean
if(gridFiles[2] != NULL)
{
if(gf->interp[index].empty == 0 &&
gf->interp[index].filled == 0)
fprintf(gridFiles[2], "-9999 ");
else
fprintf(gridFiles[2], "%f ", gf->interp[index].Zmean);
}
// Zidw
if(gridFiles[3] != NULL)
{
if(gf->interp[index].empty == 0 &&
gf->interp[index].filled == 0)
fprintf(gridFiles[3], "-9999 ");
else
fprintf(gridFiles[3], "%f ", gf->interp[index].Zidw);
}
// count
if(gridFiles[4] != NULL)
{
if(gf->interp[index].empty == 0 &&
gf->interp[index].filled == 0)
fprintf(gridFiles[4], "-9999 ");
else
fprintf(gridFiles[4], "%d ", gf->interp[index].count);
}
}
}
if(arcFiles != NULL)
for(l = 0; l < numTypes; l++)
{
if(arcFiles[l] != NULL)
fprintf(arcFiles[l], "\n");
}
if(gridFiles != NULL)
for(l = 0; l < numTypes; l++)
{
if(gridFiles[l] != NULL)
fprintf(gridFiles[l], "\n");
}
}
gf->unmap();
}
// close files
if(gridFiles != NULL)
{
for(i = 0; i < numTypes; i++)
{
if(gridFiles[i] != NULL)
fclose(gridFiles[i]);
}
}
if(arcFiles != NULL)
{
for(i = 0; i < numTypes; i++)
{
if(arcFiles[i] != NULL)
fclose(arcFiles[i]);
}
}
return 0;
}
int OutCoreInterp::finish(char *outputName, int outputFormat, unsigned int outputType)
{
int i, j;
GridPoint *p;
GridFile *gf;
int len_y;
int offset;
//struct tms tbuf;
clock_t t0, t1;
/*
// managing overlap.
read adjacent blocks and store ghost cells and update the cells
merging multiple files
*/
if(openFile != -1){
gridMap[openFile]->getGridFile()->unmap();
openFile = -1;
}
for(i = 0; i < numFiles; i++)
{
if(qlist[i].size() != 0)
{
if((gf = gridMap[i]->getGridFile()) == NULL)
{
cout << "OutCoreInterp::finish() getGridFile() NULL" << endl;
return -1;
}
gf->map();
openFile = i;
// flush UpdateInfo queue
// pop every update information
list<UpdateInfo>::const_iterator iter;
for(iter = qlist[i].begin(); iter!= qlist[i].end(); iter++){
updateInterpArray(i, (*iter).data_x, (*iter).data_y, (*iter).data_z);
}
qlist[i].erase(qlist[i].begin(), qlist[i].end());
gf->unmap();
openFile = -1;
gf = NULL;
}
}
for(i = 0; i < numFiles - 1; i++)
{
// read the upside overlap
if((gf = gridMap[i]->getGridFile()) == NULL)
{
cout << "OutCoreInterp::finish() getGridFile() NULL" << endl;
return -1;
}
if(gf->map() != -1)
{
openFile = i;
// Sriram's edit to copy over DEM values to overlap also
len_y = 2 * (gridMap[i]->getOverlapUpperBound() - gridMap[i]->getUpperBound() - 1);
if((p = (GridPoint *)malloc(sizeof(GridPoint) * len_y * GRID_SIZE_X)) == NULL)
{
cout << "OutCoreInterp::finish() malloc error" << endl;
return -1;
}
int start = (gridMap[i]->getOverlapUpperBound() - gridMap[i]->getOverlapLowerBound() - len_y) * GRID_SIZE_X;
cout << "copy from " << start << " to " << (start + len_y * GRID_SIZE_X) << endl;
memcpy(p, &(gf->interp[start]), sizeof(GridPoint) * len_y * (GRID_SIZE_X) );
gf->unmap();
gf = NULL;
openFile = -1;
} else {
cout << "OutCoreInterp::finish() gf->map() error" << endl;
return -1;
}
// update (i+1)th component
if((gf = gridMap[i+1]->getGridFile()) == NULL)
{
cout << "OutCoreInterp::finish() getGridFile() NULL" << endl;
return -1;
}
if(gf->map() != -1)
{
offset = 0;
openFile = i - 1;
for(j = 0; j < len_y * GRID_SIZE_X; j++)
{
if(gf->interp[j + offset].Zmin > p[j].Zmin)
gf->interp[j + offset].Zmin = p[j].Zmin;
if(gf->interp[j + offset].Zmax < p[j].Zmax)
gf->interp[j + offset].Zmax = p[j].Zmax;
gf->interp[j + offset].Zmean += p[j].Zmean;
gf->interp[j + offset].count += p[j].count;
if (p[j].sum != -1) {
gf->interp[j + offset].Zidw += p[j].Zidw;
gf->interp[j + offset].sum += p[j].sum;
} else {
gf->interp[j + offset].Zidw = p[j].Zidw;
gf->interp[j + offset].sum = p[j].sum;
}
}
if(p != NULL){
free(p);
p = NULL;
}
gf->unmap();
gf = NULL;
openFile = -1;
} else {
cout << "OutCoreInterp::finish() gf->map() error" << endl;
return -1;
}
}
for(i = numFiles -1; i > 0; i--)
{
// read the downside overlap
if((gf = gridMap[i]->getGridFile()) == NULL)
{
cout << "GridFile is NULL" << endl;
return -1;
}
if(gf->map() != -1)
{
openFile = i;
len_y = 2 * (gridMap[i]->getLowerBound() - gridMap[i]->getOverlapLowerBound());
if((p = (GridPoint *)malloc(sizeof(GridPoint) * len_y * GRID_SIZE_X)) == NULL)
{
cout << "OutCoreInterp::finish() malloc error" << endl;
return -1;
}
memcpy(p, &(gf->interp[0]), len_y * sizeof(GridPoint) * GRID_SIZE_X);
//finalize();
gf->unmap();
gf = NULL;
openFile = -1;
} else {
cout << "OutCoreInterp::finish gf->map() error" << endl;
return -1;
}
// update (i-1)th component
if((gf = gridMap[i-1]->getGridFile()) == NULL)
{
cout << "GridFile is NULL" << endl;
return -1;
}
if(gf->map() != -1)
{
offset = (gridMap[i-1]->getOverlapUpperBound() - gridMap[i-1]->getOverlapLowerBound() - len_y) * GRID_SIZE_X;
openFile = i - 1;
for(j = 0; j < len_y * GRID_SIZE_X; j++)
{
// Sriram - the overlap already contains the correct values
gf->interp[j + offset].Zmin = p[j].Zmin;
gf->interp[j + offset].Zmax = p[j].Zmax;
gf->interp[j + offset].Zmean = p[j].Zmean;
gf->interp[j + offset].count = p[j].count;
gf->interp[j + offset].Zidw = p[j].Zidw;
gf->interp[j + offset].sum = p[j].sum;
}
//if(i - 1 == 0)
//finalize();
if(p != NULL){
free(p);
p = NULL;
}
gf->unmap();
gf = NULL;
openFile = -1;
}
}
for(i = 0; i < numFiles; i++)
{
gridMap[i]->getGridFile()->map();
openFile = i;
finalize();
gridMap[i]->getGridFile()->unmap();
openFile = -1;
}
t0 = clock();
// merge pieces into one file
if(outputFile(outputName, outputFormat, outputType) < 0)
{
cout << "OutCoreInterp::finish outputFile error" << endl;
return -1;
}
t1 = clock();
printf("Output Execution time: %10.2f\n", (double)(t1 - t0)/CLOCKS_PER_SEC);
return 0;
}
int LocalMemoryGridFile::flush() {
trace() << " -> LocalMemoryGridFile::flush {file: " << _filename << "}" << endl;
if (!_dirty) {
// Since, there are no dirty chunks, this does not need a flush
info() << "buffers are not dirty.. need not flush {filename: " << _filename << "}" << endl;
return 0;
}
size_t bufferLen = 0;
boost::shared_array<char> buffer = createFlushBuffer(bufferLen);
if (!buffer.get() && bufferLen > 0) {
// Failed to create flush buffer
return -ENOMEM;
}
// Get the existing gridfile from GridFS to get metadata and delete the
// file from the system
try {
ScopedDbConnection dbc(globalFSOptions._connectString);
GridFS gridFS(dbc.conn(), globalFSOptions._db, globalFSOptions._collPrefix);
GridFile origGridFile = gridFS.findFile(BSON("filename" << _filename));
if (!origGridFile.exists()) {
dbc.done();
warn() << "Requested file not found for flushing back data {file: " << _filename << "}" << endl;
return -EBADF;
}
//TODO: Make checks for appropriate object correctness
//i.e. do not update anything that is not a Regular File
//Check what happens in case of a link
gridFS.removeFile(_filename);
trace() << "Removing the current file from GridFS {file: " << _filename << "}" << endl;
//TODO: Check for remove status if that was successfull or not
//TODO: Rather have an update along with active / passive flag for the
//file
try {
GridFS gridFS(dbc.conn(), globalFSOptions._db, globalFSOptions._collPrefix);
// Create an empty file to signify the file creation and open a local file for the same
trace() << "Adding new file to GridFS {file: " << _filename << "}" << endl;
BSONObj fileObj = gridFS.storeFile(buffer.get(), bufferLen, _filename);
if (!fileObj.isValid()) {
warn() << "Failed to save file object in data flush {file: " << _filename << "}" << std::endl;
dbc.done();
return -EBADF;
}
// Update the last updated date for the document
BSONObj metadata = origGridFile.getMetadata();
BSONElement fileObjId = fileObj.getField("_id");
dbc->update(globalFSOptions._filesNS, BSON("_id" << fileObjId.OID()),
BSON("$set" << BSON(
"uploadDate" << origGridFile.getUploadDate()
<< "metadata.type" << "file"
<< "metadata.filename" << mgridfs::getPathBasename(_filename)
<< "metadata.directory" << mgridfs::getPathDirname(_filename)
<< "metadata.lastUpdated" << jsTime()
<< "metadata.uid" << metadata["uid"]
<< "metadata.gid" << metadata["gid"]
<< "metadata.mode" << metadata["mode"]
)
)
);
} catch (DBException& e) {
error() << "Caught exception in saving remote file in flush {code: " << e.getCode() << ", what: " << e.what()
<< ", exception: " << e.toString() << "}" << endl;
return -EIO;
}
dbc.done();
} catch (DBException& e) {
// Something failed in getting the file from GridFS
error() << "Caught exception in getting remote file for flush {code: " << e.getCode() << ", what: " << e.what()
<< ", exception: " << e.toString() << "}" << endl;
return -EIO;
}
_dirty = false;
debug() << "Completed flushing the file content to GridFS {file: " << _filename << "}" << endl;
return 0;
}
int
main(int argc, char **argv)
{
ArgumentParser argp(argc, argv, "ho:fd:c:");
if (argp.has_arg("h")) {
print_usage(argv[0]);
exit(0);
}
const std::vector<const char *> &items = argp.items();
std::string output_dir = "tmp/";
std::string database = "fflog";
std::string collection;
std::string query_coll;
bool filename_indexed = ! argp.has_arg("f");
std::vector<std::pair<long long, long long> > times;
if (argp.has_arg("o")) {
output_dir = argp.arg("o");
if (output_dir[output_dir.length() - 1] != '/') {
output_dir += "/";
}
}
if (argp.has_arg("d")) {
database = argp.arg("d");
}
if (argp.has_arg("c")) {
collection = argp.arg("c");
} else {
print_usage(argv[0]);
printf("No collection given\n");
exit(-1);
}
query_coll = database + "." + collection;
if (items.empty()) {
times.push_back(std::make_pair(0L, std::numeric_limits<long long>::max()));
} else {
for (unsigned int i = 0; i < items.size(); ++i) {
std::string item = items[i];
std::string::size_type dotpos = item.find("..");
if (dotpos == std::string::npos) {
// singular timestamp
long int ts = argp.parse_item_int(i);
times.push_back(std::make_pair(ts, ts));
} else {
// range
std::string first_ts, second_ts;
first_ts = item.substr(0, dotpos);
second_ts = item.substr(dotpos + 2);
times.push_back(std::make_pair(StringConversions::to_long(first_ts),
StringConversions::to_long(second_ts)));
}
}
}
unsigned int image_n = 0;
DBClientConnection *mongodb_client =
new DBClientConnection(/* auto reconnect */ true);
std::string errmsg;
mongodb_client->connect("localhost", errmsg);
GridFS *gridfs = new GridFS(*mongodb_client, "fflog");
for (unsigned int i = 0; i < times.size(); ++i) {
Query q;
if (times[i].first == times[i].second) {
printf("Querying for timestamp %lli\n", times[i].first);
q = QUERY("timestamp" << times[i].first).sort("timestamp", 1);
} else {
printf("Querying for range %lli..%lli\n", times[i].first, times[i].second);
q = QUERY("timestamp"
<< mongo::GTE << times[i].first
<< mongo::LTE << times[i].second)
.sort("timestamp", 1);
}
#if __cplusplus >= 201103L
std::unique_ptr<mongo::DBClientCursor> cursor =
mongodb_client->query(query_coll, q);
#else
std::auto_ptr<mongo::DBClientCursor> cursor =
mongodb_client->query(query_coll, q);
#endif
while (cursor->more()) {
BSONObj doc = cursor->next();
BSONObj imgdoc = doc.getObjectField("image");
if (imgdoc["colorspace"].String() == "RGB") {
std::string filename = imgdoc.getFieldDotted("data.filename").String();
long filesize = imgdoc.getFieldDotted("data.length").numberLong();
std::string image_id = imgdoc["image_id"].String();
std::string out_filename;
char *fntmp;
if (filename_indexed) {
if (asprintf(&fntmp, "%s%s-%08d.png", output_dir.c_str(),
image_id.c_str(), image_n++) != -1)
{
out_filename = fntmp;
free(fntmp);
}
} else {
if (asprintf(&fntmp, "%s%s.png", output_dir.c_str(),
filename.c_str()) != -1)
{
out_filename = fntmp;
free(fntmp);
}
++image_n;
}
printf("Restoring RGB image %s (%s)\n", filename.c_str(), out_filename.c_str());
GridFile file = gridfs->findFile(filename);
if (! file.exists()) {
printf("File %s does not exist\n", filename.c_str());
continue;
}
unsigned int width = imgdoc["width"].Int();
unsigned int height = imgdoc["height"].Int();
if (colorspace_buffer_size(RGB, width, height) != (size_t)filesize) {
printf("Buffer size mismatch (DB %li vs. exp. %zu)\n",
filesize, colorspace_buffer_size(RGB, width, height));
continue;
}
unsigned char *buffer = malloc_buffer(RGB, width, height);
unsigned char *tmp = buffer;
for (int c = 0; c < file.getNumChunks(); ++c) {
mongo::GridFSChunk chunk = file.getChunk(c);
int len = 0;
const char *chunk_data = chunk.data(len);
memcpy(tmp, chunk_data, len);
tmp += len;
}
PNGWriter writer(out_filename.c_str(), width, height);
writer.set_buffer(RGB, buffer);
writer.write();
free(buffer);
}
}
}
delete gridfs;
delete mongodb_client;
}
