cafs::link cafs::import( const fc::path& p ) {
if( !fc::exists(p) ) {
FC_THROW_MSG( "Path does not exist %s", p.string() );
}
if( fc::is_regular_file( p ) ) {
if( fc::file_size(p) > IMBED_THRESHOLD ) {
auto file_head = import_file(p);
fc::vector<char> data(MAX_CHUNK_SIZE);
fc::datastream<char*> ds(data.data()+1, data.size()-1);
fc::raw::pack( ds, file_head );
data[0] = cafs::file_header_type;
//fc::datastream<const char*> ds2(data.data()+1, data.size()-1);
//file_header tmp;
//fc::raw::unpack( ds2, tmp );
//slog( "test unpack %s", fc::json::to_string( tmp ).c_str() );
data.resize( ds.tellp() );
//slog( "pre randomized... '%s'", fc::to_hex( data.data(), 16 ).c_str() );
size_t seed = randomize(data, *((uint64_t*)fc::sha1::hash(data.data(),data.size()).data()) );
auto chunk_head = slice_chunk( data );
//slog( "slice chunk %s", fc::json::to_string( chunk_head ).c_str() );
store_chunk( chunk_head, data );
return link( chunk_head.calculate_id(), seed );
} else { // no header, just raw data from the file stored in the chunk
fc::vector<char> data( fc::file_size(p)+1 );
data[0] = file_data_type;
fc::ifstream ifile( p.string(), fc::ifstream::binary );
ifile.read( data.data()+1, data.size()-1 );
size_t seed = randomize(data, *((uint64_t*)fc::sha1::hash(data.data(),data.size()).data()) );
auto chunk_head = slice_chunk( data );
store_chunk( chunk_head, data );
return link( chunk_head.calculate_id(), seed );
}
}
else if( fc::is_directory(p) ) {
auto dir = import_directory(p);
fc::vector<char> data(MAX_CHUNK_SIZE);
fc::datastream<char*> ds(data.data()+1, data.size()-1);
fc::raw::pack( ds, dir );
data[0] = directory_type;
data.resize( ds.tellp()+1 );
size_t seed = randomize(data, *((uint64_t*)fc::sha1::hash(data.data(),data.size()).data()) );
auto chunk_head = slice_chunk( data );
link l( chunk_head.calculate_id(), seed );
store_chunk( chunk_head, data );
return l;
}
FC_THROW_MSG( "Unsupported file type while importing '%s'", p.string() );
return cafs::link();
}
static JSBool
do_import(JSContext *context,
JSObject *obj,
Importer *priv,
const char *name)
{
char *filename;
char *full_path;
char *dirname = NULL;
jsval search_path_val;
JSObject *search_path;
JSObject *module_obj = NULL;
guint32 search_path_len;
guint32 i;
JSBool result;
GPtrArray *directories;
jsid search_path_name;
search_path_name = gjs_runtime_get_const_string(JS_GetRuntime(context),
GJS_STRING_SEARCH_PATH);
if (!gjs_object_require_property(context, obj, "importer", search_path_name, &search_path_val)) {
return JS_FALSE;
}
if (!JSVAL_IS_OBJECT(search_path_val)) {
gjs_throw(context, "searchPath property on importer is not an object");
return JS_FALSE;
}
search_path = JSVAL_TO_OBJECT(search_path_val);
if (!JS_IsArrayObject(context, search_path)) {
gjs_throw(context, "searchPath property on importer is not an array");
return JS_FALSE;
}
if (!JS_GetArrayLength(context, search_path, &search_path_len)) {
gjs_throw(context, "searchPath array has no length");
return JS_FALSE;
}
result = JS_FALSE;
filename = g_strdup_printf("%s.js", name);
full_path = NULL;
directories = NULL;
/* First try importing an internal module like byteArray */
if (priv->is_root &&
gjs_is_registered_native_module(context, obj, name) &&
import_native_file(context, obj, name)) {
gjs_debug(GJS_DEBUG_IMPORTER,
"successfully imported module '%s'", name);
result = JS_TRUE;
goto out;
}
for (i = 0; i < search_path_len; ++i) {
jsval elem;
elem = JSVAL_VOID;
if (!JS_GetElement(context, search_path, i, &elem)) {
/* this means there was an exception, while elem == JSVAL_VOID
* means no element found
*/
goto out;
}
if (JSVAL_IS_VOID(elem))
continue;
if (!JSVAL_IS_STRING(elem)) {
gjs_throw(context, "importer searchPath contains non-string");
goto out;
}
g_free(dirname);
dirname = NULL;
if (!gjs_string_to_utf8(context, elem, &dirname))
goto out; /* Error message already set */
/* Ignore empty path elements */
if (dirname[0] == '\0')
continue;
/* Try importing __init__.js and loading the symbol from it */
if (full_path)
g_free(full_path);
full_path = g_build_filename(dirname, MODULE_INIT_FILENAME,
NULL);
module_obj = load_module_init(context, obj, full_path);
if (module_obj != NULL) {
jsval obj_val;
if (JS_GetProperty(context,
module_obj,
name,
&obj_val)) {
if (!JSVAL_IS_VOID(obj_val) &&
JS_DefineProperty(context, obj,
name, obj_val,
NULL, NULL,
GJS_MODULE_PROP_FLAGS & ~JSPROP_PERMANENT)) {
result = JS_TRUE;
goto out;
}
}
}
/* Second try importing a directory (a sub-importer) */
if (full_path)
g_free(full_path);
full_path = g_build_filename(dirname, name,
NULL);
if (g_file_test(full_path, G_FILE_TEST_IS_DIR)) {
gjs_debug(GJS_DEBUG_IMPORTER,
"Adding directory '%s' to child importer '%s'",
full_path, name);
if (directories == NULL) {
directories = g_ptr_array_new();
}
g_ptr_array_add(directories, full_path);
/* don't free it twice - pass ownership to ptr array */
full_path = NULL;
}
/* If we just added to directories, we know we don't need to
* check for a file. If we added to directories on an earlier
* iteration, we want to ignore any files later in the
* path. So, always skip the rest of the loop block if we have
* directories.
*/
if (directories != NULL) {
continue;
}
/* Third, if it's not a directory, try importing a file */
g_free(full_path);
full_path = g_build_filename(dirname, filename,
NULL);
if (g_file_test(full_path, G_FILE_TEST_EXISTS)) {
if (import_file(context, obj, name, full_path)) {
gjs_debug(GJS_DEBUG_IMPORTER,
"successfully imported module '%s'", name);
result = JS_TRUE;
}
/* Don't keep searching path if we fail to load the file for
* reasons other than it doesn't exist... i.e. broken files
* block searching for nonbroken ones
*/
goto out;
}
gjs_debug(GJS_DEBUG_IMPORTER,
"JS import '%s' not found in %s",
name, dirname);
}
if (directories != NULL) {
/* NULL-terminate the char** */
g_ptr_array_add(directories, NULL);
if (import_directory(context, obj, name,
(const char**) directories->pdata)) {
gjs_debug(GJS_DEBUG_IMPORTER,
"successfully imported directory '%s'", name);
result = JS_TRUE;
}
}
out:
if (directories != NULL) {
char **str_array;
/* NULL-terminate the char**
* (maybe for a second time, but doesn't matter)
*/
g_ptr_array_add(directories, NULL);
str_array = (char**) directories->pdata;
g_ptr_array_free(directories, FALSE);
g_strfreev(str_array);
}
g_free(full_path);
g_free(filename);
g_free(dirname);
if (!result &&
!JS_IsExceptionPending(context)) {
/* If no exception occurred, the problem is just that we got to the
* end of the path. Be sure an exception is set.
*/
gjs_throw(context, "No JS module '%s' found in search path", name);
}
return result;
}
static JSBool
do_import(JSContext *context,
JSObject *obj,
Importer *priv,
std::string &name)
{
std::string filename;
std::string full_path;
std::string dirname;
jsval search_path_val;
JSObject *search_path;
JSObject *module_obj = NULL;
uint32_t search_path_len;
uint32_t i;
JSBool result;
std::vector<std::string> directories;
jsid search_path_name;
bool exists;
search_path_name = gjs_context_get_const_string(context, GJS_STRING_SEARCH_PATH);
if (!gjs_object_require_property(context, obj, "importer", search_path_name, &search_path_val)) {
return JS_FALSE;
}
if (!search_path_val.isObject()) {
gjs_throw(context, "searchPath property on importer is not an object");
return JS_FALSE;
}
search_path = JSVAL_TO_OBJECT(search_path_val);
if (!JS_IsArrayObject(context, search_path)) {
gjs_throw(context, "searchPath property on importer is not an array");
return JS_FALSE;
}
if (!JS_GetArrayLength(context, search_path, &search_path_len)) {
gjs_throw(context, "searchPath array has no length");
return JS_FALSE;
}
result = JS_FALSE;
filename = std::string(name) + ".js";
/* First try importing an internal module like byteArray */
if (priv->is_root &&
gjs_is_registered_native_module(context, obj, name) &&
import_native_file(context, obj, name)) {
gjs_debug(GJS_DEBUG_IMPORTER,
"successfully imported module '%s'", name);
result = JS_TRUE;
goto out;
}
for (i = 0; i < search_path_len; ++i) {
jsval elem;
elem = JSVAL_VOID;
if (!JS_GetElement(context, search_path, i, &elem)) {
/* this means there was an exception, while elem == JSVAL_VOID
* means no element found
*/
goto out;
}
if (JSVAL_IS_VOID(elem))
continue;
if (!JSVAL_IS_STRING(elem)) {
gjs_throw(context, "importer searchPath contains non-string");
goto out;
}
if (!gjs_string_to_utf8(context, elem, dirname))
goto out; /* Error message already set */
/* Ignore empty path elements */
if (dirname[0] == '\0')
continue;
/* Try importing __init__.js and loading the symbol from it */
full_path = pathCombine(dirname, MODULE_INIT_FILENAME);
module_obj = load_module_init(context, obj, full_path);
if (module_obj != NULL) {
jsval obj_val;
if (JS_GetProperty(context,
module_obj,
name.c_str(),
&obj_val)) {
if (!JSVAL_IS_VOID(obj_val) &&
JS_DefineProperty(context, obj,
name.c_str(), obj_val,
NULL, NULL,
GJS_MODULE_PROP_FLAGS & ~JSPROP_PERMANENT)) {
result = JS_TRUE;
goto out;
}
}
}
/* Second try importing a directory (a sub-importer) */
full_path = pathCombine(dirname, name);
if (is_directory(full_path)) {
std::cout << "Adding directory '" << full_path << "' to child importer '" << name << "'\n",
directories.push_back(full_path);
}
/* If we just added to directories, we know we don't need to
* check for a file. If we added to directories on an earlier
* iteration, we want to ignore any files later in the
* path. So, always skip the rest of the loop block if we have
* directories.
*/
if (directories.size() > 0) {
continue;
}
/* Third, if it's not a directory, try importing a file */
full_path = pathCombine(dirname, filename);
std::cout << "full path: " << full_path << "\n";
exists = is_regular(full_path);
if (!exists) {
std::cout << "JS import '" << name << "' not found in " << dirname << "\n";
continue;
}
if (import_file_on_module (context, obj, name, full_path)) {
std::cout << "successfully imported module '" << name << "'\n";
result = JS_TRUE;
}
/* Don't keep searching path if we fail to load the file for
* reasons other than it doesn't exist... i.e. broken files
* block searching for nonbroken ones
*/
goto out;
}
if (directories.size() > 0) {
/* NULL-terminate the char** */
if (import_directory(context, obj, name, directories)) {
std::cout << "successfully imported directory '" << name << "'\n";
result = JS_TRUE;
}
}
out:
if (!result &&
!JS_IsExceptionPending(context)) {
/* If no exception occurred, the problem is just that we got to the
* end of the path. Be sure an exception is set.
*/
gjs_throw(context, "No JS module '%s' found in search path", name.c_str());
}
return result;
}
/**
* Imports the files in a directory and returns a directory object.
*/
cafs::directory cafs::import_directory( const fc::path& p ) {
directory dir;
fc::directory_iterator itr(p);
fc::directory_iterator end;
chunk_header cur_chunk;
fc::vector<char> cur_chunk_data(MAX_CHUNK_SIZE);
fc::datastream<char*> cur_chunk_ds(cur_chunk_data.data(),cur_chunk_data.size());
while( itr != end ) {
fc::path cur_path = *itr;
slog( "%s", cur_path.generic_string().c_str() );
fc::string name = cur_path.filename().string();
file_type cur_file_type = unknown;
fc::vector<char> cur_file;
if( fc::is_directory( cur_path ) && name[0] != '.' ) {
directory d = fc::async( [=]() { return import_directory( cur_path ); } );
cur_file = fc::raw::pack( d );
cur_file_type = directory_type;
} else if( fc::is_regular_file( cur_path ) ) {
if( fc::file_size(cur_path) > IMBED_THRESHOLD ) {
cur_file = fc::raw::pack( import_file(cur_path) );
cur_file_type = file_header_type;
} else {
cur_file.resize( fc::file_size(cur_path) );
fc::ifstream inf( cur_path, fc::ifstream::binary );
inf.read(cur_file.data(), cur_file.size() );
cur_file_type = file_data_type;
}
}
++itr;
fc::sha1 h = fc::sha1::hash(cur_file.data(),cur_file.size());
fc::optional<file_ref> ofr = my->file_db.fetch(h);
//slog( " %s", fc::string(h).c_str() );
if( ofr ) {
// wlog( "We already have a file ref for this..." );
dir.entries.push_back( directory::entry(name) );
dir.entries.back().ref = *ofr;
}
if( !ofr && !add_to_chunk( cur_chunk_ds, cur_file, name, dir, cur_file_type ) ) {
// save cur chunk...
// wlog( " save chunk size %llu", cur_chunk_ds.tellp() );
cur_chunk_data.resize(cur_chunk_ds.tellp());
save_chunk( *this, dir, cur_chunk_data );
cur_chunk_data.resize(MAX_CHUNK_SIZE);
cur_chunk_ds = fc::datastream<char*>(cur_chunk_data.data(),
cur_chunk_data.size());
add_to_chunk( cur_chunk_ds, cur_file, name, dir, cur_file_type );
}
if( itr == end && cur_chunk_ds.tellp() > 0 ) {
// wlog( " save chunk size %llu", cur_chunk_ds.tellp() );
cur_chunk_data.resize(cur_chunk_ds.tellp());
save_chunk( *this, dir, cur_chunk_data );
}
}
for( auto itr = dir.entries.begin(); itr != dir.entries.end(); ++itr ) {
my->file_db.store( itr->ref );
}
return dir;
}
void import_mailbox(TALLOC_CTX *mem_ctx,
struct mapi_session *session,
struct mbox_data *mdata)
{
enum MAPISTATUS retval;
struct dirent *direntp;
DIR *dirp;
mapi_object_t obj_store;
char *base_path;
mdata->start_time = time(NULL);
base_path = talloc_asprintf(mem_ctx, "%s/%s", DEFAULT_EXPORT_PATH, mdata->username);
if (!base_path) {
goto fail;
}
/* Open systemfolder database */
mdata->tdb_sysfolder = tdb_open_database(mdata, base_path, TDB_SYSFOLDER);
if (!mdata->tdb_sysfolder) {
DEBUG(0, ("[!] Error opening system folder db\n"));
goto fail;
}
/* Open PidTagFolderID to FolderName database */
mdata->tdb_foldermap = tdb_open_database(mdata, base_path, TDB_FOLDERMAP);
if (!mdata->tdb_foldermap) {
DEBUG(0, ("[!] Error opening id mapping db\n"));
goto fail;
}
/* Open the folder */
dirp = opendir(base_path);
if (!dirp) {
DEBUG(0, ("[!] Error opening directory %s: %s (%d)\n",
base_path, strerror(errno), errno));
goto fail;
}
/* Open the root folder */
mapi_object_init(&obj_store);
retval = OpenUserMailbox(session, mdata->username, &obj_store);
if (retval != MAPI_E_SUCCESS) {
const char *error = mapi_get_errstr(GetLastError());
DEBUG(0, ("[!] OpenUserMailbox: %s\n", error));
goto fail;
}
/* Import the directory */
while ((direntp = readdir(dirp)) != NULL) {
if (strncasecmp(direntp->d_name, "0x", 2) == 0) {
/* This is the root folder */
char *path = talloc_asprintf(mem_ctx, "%s/%s", base_path, direntp->d_name);
retval = import_directory(mem_ctx, mdata, &obj_store, NULL, path);
if (retval != MAPI_E_SUCCESS) {
DEBUG(0, ("import_directory failed with %s\n",
mapi_get_errstr(GetLastError())));
break;
}
talloc_free(path);
break;
}
}
fail:
mapi_object_release(&obj_store);
if (dirp)
closedir(dirp);
if (mdata->tdb_foldermap) {
tdb_close(mdata->tdb_foldermap);
mdata->tdb_foldermap = NULL;
}
if (mdata->tdb_sysfolder) {
tdb_close(mdata->tdb_sysfolder);
mdata->tdb_sysfolder = NULL;
}
if (base_path)
talloc_free(base_path);
mdata->end_time = time(NULL);
return;
}
static enum MAPISTATUS import_directory(TALLOC_CTX *mem_ctx,
const struct mbox_data *mdata,
mapi_object_t *obj_store,
mapi_object_t *obj_parent,
const char *base_path)
{
enum MAPISTATUS retval;
DIR *dirp;
struct dirent *direntp;
mapi_object_t obj_folder;
mapi_object_t obj_inbox;
mapi_object_t obj_child;
mapi_id_t id_folder;
char *folder_id;
char *olFolderSrc;
struct SPropTagArray *SPropTagArray;
struct SPropValue *lpProps;
uint32_t cValues = 0;
struct SRow aRow;
DEBUG(5,("[*] Importing directory %s\n", base_path));
/* Open the filesystem folder */
dirp = opendir(base_path);
if (!dirp) {
DEBUG(0, ("[!] Error opening directory %s: %s (%d)\n",
base_path, strerror(errno), errno));
return MAPI_E_NOT_FOUND; // TODO map to proper code
}
mapi_object_init(&obj_folder);
mapi_object_init(&obj_child);
mapi_object_init(&obj_inbox);
/* I want to get the folder ID from the remote Exchange server and
check in the systemfolder database if it matches with something.
we can get the remote folder ID from the directory name */
folder_id = import_get_folder_id(base_path);
if (!folder_id) {
DEBUG(0, ("[!] Error getting folder ID from directory name\n"));
return MAPI_E_NOT_FOUND; // TODO map to proper code
}
if (!obj_parent) {
DEBUG(5, ("parent is null\n"));
retval = GetDefaultFolder(obj_store, &id_folder, olFolderInbox);
if (retval != MAPI_E_SUCCESS) {
DEBUG(0, ("[!] GetDefaultFolder: %s\n", mapi_get_errstr(GetLastError())));
return retval;
}
DEBUG(4, ("[*] Opening folder %u\n", olFolderInbox));
retval = OpenFolder(obj_store, id_folder, &obj_inbox);
if (retval != MAPI_E_SUCCESS) {
DEBUG(0, ("[!] OpenFolder: %s\n", mapi_get_errstr(GetLastError())));
return retval;
}
obj_parent = &obj_inbox;
}
/* XXX Begin of hack */
#if 0
olFolderSrc = import_is_system_folder(mdata, folder_id);
if (!olFolderSrc) {
DEBUG(5, ("[*] Not system folder, skip\n"));
talloc_free(olFolderSrc);
return MAPI_E_SUCCESS;
}
uint32_t olFolder = atoi(olFolderSrc);
talloc_free(olFolderSrc);
retval = MAPI_E_SUCCESS;
if (olFolder == olFolderContacts) {
retval = GetDefaultFolder(obj_store, &id_folder, olFolderContacts);
} else if (olFolder == olFolderCalendar) {
retval = GetDefaultFolder(obj_store, &id_folder, olFolderCalendar);
} else if (olFolder == olFolderTopInformationStore) {
retval = GetDefaultFolder(obj_store, &id_folder, olFolderInbox);
}
if (retval != MAPI_E_SUCCESS) {
DEBUG(0, ("[!] GetDefaultFolder: %s\n", mapi_get_errstr(GetLastError())));
return retval;
}
DEBUG(4, ("Opening folder %u\n", olFolder));
retval = OpenFolder(obj_store, id_folder, &obj_folder);
if (retval != MAPI_E_SUCCESS) {
DEBUG(0, ("[!] OpenFolder: %s\n", mapi_get_errstr(GetLastError())));
return retval;
}
#endif
/* XXX end of hack */
#if 1
olFolderSrc = import_is_system_folder(mdata, folder_id);
if (olFolderSrc) {
char *folder_name = import_get_folder_name(mdata, folder_id);
if (folder_name) {
DEBUG(5, ("[*] Origin Folder '%s' mapped to name '%s'\n", folder_id, folder_name));
talloc_free(folder_name);
folder_name = NULL;
}
/* This is a system folder, then I am calling GetDefaultFolder
to retrieve the id then I open the folder */
uint32_t olFolder = atoi(olFolderSrc);
talloc_free(olFolderSrc);
retval = GetDefaultFolder(obj_store, &id_folder, olFolder);
if (retval != MAPI_E_SUCCESS) {
DEBUG(0, ("[!] GetDefaultFolder: %s\n", mapi_get_errstr(GetLastError())));
return retval;
}
DEBUG(4, ("[*] Opening folder %u\n", olFolder));
retval = OpenFolder(obj_store, id_folder, &obj_folder);
if (retval != MAPI_E_SUCCESS) {
DEBUG(0, ("[!] OpenFolder: %s\n", mapi_get_errstr(GetLastError())));
return retval;
}
SPropTagArray = set_SPropTagArray(mem_ctx, 0x1, PidTagDisplayName);
retval = GetProps(&obj_folder, MAPI_UNICODE, SPropTagArray, &lpProps, &cValues);
MAPIFreeBuffer(SPropTagArray);
if (retval == MAPI_E_SUCCESS) {
aRow.cValues = cValues;
aRow.lpProps = lpProps;
folder_name = (char *) find_SPropValue_data(&aRow, PidTagDisplayName);
if (folder_name) {
DEBUG(5, ("[*] Destination Folder: '%s'\n", folder_name));
}
}
} else {
/* this is not a system folder, I know what is the root base where
need to create it i and open it */
char *folder_name = import_get_folder_name(mdata, folder_id);
if (!folder_name) {
DEBUG(0, ("[!] Invalid Folder Name\n"));
return MAPI_E_INVALID_PARAMETER;
}
DEBUG(4, ("[*] Creating folder %s\n", folder_name));
retval = CreateFolder(obj_parent, FOLDER_GENERIC, folder_name,
NULL, OPEN_IF_EXISTS|MAPI_UNICODE, &obj_folder);
if (retval != MAPI_E_SUCCESS) {
DEBUG(0, ("[!] CreateFolder: %s\n", mapi_get_errstr(GetLastError())));
talloc_free(folder_name);
return retval;
}
talloc_free(folder_name);
return MAPI_E_SUCCESS;
}
#endif
/* Import the files and clildren folders */
while ((direntp = readdir(dirp)) != NULL) {
if (strcmp(direntp->d_name, ".") == 0) {
continue;
}
if (strcmp(direntp->d_name, "..") == 0) {
continue;
}
char *ext = strrchr(direntp->d_name, '.');
if (!ext) {
if (strncasecmp(direntp->d_name, "0x", 2) == 0) {
char *child_path = talloc_asprintf(mem_ctx, "%s/%s", base_path, direntp->d_name);
retval = import_directory(mem_ctx, mdata, obj_store, &obj_folder, child_path);
if (retval != MAPI_E_SUCCESS) {
DEBUG(0, ("import_directory failed with %s\n", mapi_get_errstr(GetLastError())));
talloc_free(child_path);
return retval;
}
talloc_free(child_path);
}
continue;
}
if (strncasecmp(ext, ".ocpf", 5) == 0) {
char *child_path = talloc_asprintf(mem_ctx, "%s/%s", base_path, direntp->d_name);
import_ocpf_file(mem_ctx, mdata, obj_store, &obj_folder, child_path);
talloc_free(child_path);
}
}
mapi_object_release(&obj_folder);
mapi_object_release(&obj_inbox);
/* Close directory */
closedir(dirp);
return retval;
}
