void TorrentFilesModel::ClearEmptyParents (boost::filesystem::path path)
{
const auto pos = Path2Node_.find (path);
if (pos == Path2Node_.end ())
{
qWarning () << Q_FUNC_INFO
<< "unknown path"
<< path.c_str ();
return;
}
const auto& node = pos->second;
if (!node->IsEmpty ())
{
UpdateSizeGraph (RootNode_);
return;
}
const auto& parentNode = node->GetParent ();
const auto nodeRow = node->GetRow ();
const auto& parentIndex = FindIndex (path.branch_path ());
beginRemoveRows (parentIndex, nodeRow, nodeRow);
parentNode->EraseChild (parentNode->begin () + nodeRow);
Path2Node_.erase (pos);
endRemoveRows ();
ClearEmptyParents (path.branch_path ());
}
static void _createDirectories(Transport::Config *config, boost::filesystem::path ph) {
if (ph.empty() || exists(ph)) {
return;
}
// First create branch, by calling ourself recursively
_createDirectories(config, ph.branch_path());
// Now that parent's path exists, create the directory
boost::filesystem::create_directory(ph);
#ifndef WIN32
if (!CONFIG_STRING(config, "service.group").empty() && !CONFIG_STRING(config, "service.user").empty()) {
struct group *gr;
if ((gr = getgrnam(CONFIG_STRING(config, "service.group").c_str())) == NULL) {
std::cerr << "Invalid service.group name " << CONFIG_STRING(config, "service.group") << "\n";
}
struct passwd *pw;
if ((pw = getpwnam(CONFIG_STRING(config, "service.user").c_str())) == NULL) {
std::cerr << "Invalid service.user name " << CONFIG_STRING(config, "service.user") << "\n";
}
chown(ph.string().c_str(), pw->pw_uid, gr->gr_gid);
}
#endif
}
void create_parents( const fs::path &dpath )
{
string err( "create_parent(): " );
fs::path branch( dpath.branch_path() );
string who("create_parents");
if( dpath.empty() ) {
err.append( "cannot create an empty path." );
throw CannotCreateParents( err );
}
else if( !exists(dpath) ) {
if( branch.empty() ) create_directory( dpath );
else if( !exists(branch) ) {
create_parents( branch );
create_directory( dpath );
}
else if( is_directory(branch) ) create_directory( dpath );
else {
err.append( branch.native_file_string() ); err.append( " is not a directory." );
throw CannotCreateParents( err );
}
}
else if( !is_directory(dpath) ) {
err.append( dpath.native_file_string() ); err.append( " is not a directory." );
throw CannotCreateParents( err );
}
return;
}
inline SAGA_EXPORT std::string parent(boost::filesystem::path const& p)
{
#if BOOST_VERSION >= 103600
return p.parent_path().string();
#else
return p.branch_path().string();
#endif
}
//! get the relative file path from the host directory
static std::string get_rel_file_path(const fs::path &file){
fs::path abs_path = file.branch_path();
fs::path rel_path = file.leaf();
while (not abs_path.empty() and abs_path.leaf() != "host"){
rel_path = abs_path.leaf() / rel_path;
abs_path = abs_path.branch_path();
}
return rel_path.string();
}
fs::path correctPathCase(fs::path Path) {
using namespace boost::filesystem;
#ifndef CASE_SENSITIVE_FILESYSTEM
if (!exists(Path))
return path();
return Path;
#else
// If the path is OK as it stands, do nothing.
if (exists(Path)) return Path;
// If the path doesn't seem to be OK, track backwards through it
// looking for the point at which the problem first arises. Path
// will contain the parts of the path that exist on the current
// filesystem, and pathElts will contain the parts after that point,
// which may have incorrect case.
stack<string> pathElts;
while (!Path.empty() && !exists(Path)) {
pathElts.push(Path.filename().string());
Path = Path.branch_path();
}
// Now proceed forwards through the possibly-incorrect elements.
while (!pathElts.empty()) {
// Does this element need to be a directory?
// (If we are searching for /foo/bar/baz, and /foo contains a file
// bar and a directory Bar, then we need to know which is the one
// we're looking for. This will still be unreliable if /foo
// contains directories bar and Bar, but a full backtracking
// search would be complicated; for now this should be adequate!)
const bool needDir = pathElts.size() > 1;
string elt(pathElts.top());
pathElts.pop();
// Does this element exist?
if (exists(Path/elt) && (!needDir || is_directory(Path/elt))) {
// If so, use it.
Path /= elt;
} else {
// If not, search for a suitable candidate.
to_upper(elt);
directory_iterator end;
bool found = false;
for (directory_iterator dir(Path); dir != end; ++dir) {
string uleaf = dir->path().filename().string();
to_upper(uleaf);
if (uleaf == elt && (!needDir || is_directory(*dir))) {
Path /= dir->path().filename();
found = true;
break;
}
}
if (!found) return "";
}
}
return Path.string();
#endif
}
boost::filesystem::path ensureParentDirCreated(const boost::filesystem::path& p) {
const boost::filesystem::path parent = p.branch_path();
if (!boost::filesystem::exists(parent)) {
ensureParentDirCreated(parent);
log() << "creating directory " << parent.string() << endl;
boost::filesystem::create_directory(parent);
flushMyDirectory(parent); // flushes grandparent to ensure parent exists after crash
}
verify(boost::filesystem::is_directory(parent));
return parent;
}
void flushMyDirectory(const boost::filesystem::path& file) {
#ifdef __linux__ // this isn't needed elsewhere
static bool _warnedAboutFilesystem = false;
// if called without a fully qualified path it asserts; that makes mongoperf fail.
// so make a warning. need a better solution longer term.
// massert(13652, str::stream() << "Couldn't find parent dir for file: " << file.string(),);
if (!file.has_branch_path()) {
log() << "warning flushMyDirectory couldn't find parent dir for file: "
<< file.string();
return;
}
boost::filesystem::path dir = file.branch_path(); // parent_path in new boosts
LOG(1) << "flushing directory " << dir.string();
int fd = ::open(dir.string().c_str(), O_RDONLY); // DO NOT THROW OR ASSERT BEFORE CLOSING
massert(13650, str::stream() << "Couldn't open directory '" << dir.string()
<< "' for flushing: " << errnoWithDescription(),
fd >= 0);
if (fsync(fd) != 0) {
int e = errno;
if (e == EINVAL) { // indicates filesystem does not support synchronization
if (!_warnedAboutFilesystem) {
log() << "\tWARNING: This file system is not supported. For further information"
<< " see:"
<< startupWarningsLog;
log() << "\t\t\thttp://dochub.mongodb.org/core/unsupported-filesystems"
<< startupWarningsLog;
log() << "\t\tPlease notify MongoDB, Inc. if an unlisted filesystem generated "
<< "this warning." << startupWarningsLog;
_warnedAboutFilesystem = true;
}
}
else {
close(fd);
massert(13651, str::stream() << "Couldn't fsync directory '" << dir.string()
<< "': " << errnoWithDescription(e),
false);
}
}
close(fd);
#endif
}
void TorrentFilesModel::handleFileRenamed (int torrent, int file, const QString& newName)
{
if (torrent != Index_)
return;
const auto filePos = std::find_if (Path2Node_.begin (), Path2Node_.end (),
[file] (const Path2Node_t::value_type& pair)
{ return pair.second->FileIndex_ == file; });
if (filePos == Path2Node_.end ())
{
qWarning () << Q_FUNC_INFO
<< "unknown file index"
<< file
<< "for torrent"
<< torrent
<< "was renamed to"
<< newName;
return;
}
const auto node = filePos->second;
ClearEmptyParents (filePos->first);
const boost::filesystem::path newPath { newName.toUtf8 ().constData () };
const auto& parentNode = MkParentIfDoesntExist (newPath, true);
node->Name_ = QString::fromUtf8 (newPath.leaf ().string ().c_str ());
node->Reparent (parentNode);
beginInsertRows (FindIndex (newPath.branch_path ()), parentNode->GetRowCount (), parentNode->GetRowCount ());
Path2Node_ [newPath] = node;
parentNode->AppendExisting (node);
endInsertRows ();
UpdateSizeGraph (RootNode_);
}
/**
* Non-Windows shared library constructor. This ensures that the environment
* variable \c TWEEK_BASE_DIR is set as soon as this shared library is loaded.
* If it is not set, then it sets it based on an assumption about the
* structure of a Tweek installation. More specifically, an assumption is made
* that this shared library lives in the \c lib subdirectory of the Tweek
* installation. Therefore, the root of the Tweek installation is the parent
* of the directory containing this shared library.
*/
extern "C" void __attribute ((constructor)) tweekLibraryInit()
{
fs::path base_dir;
const char* env_dir = std::getenv("TWEEK_BASE_DIR");
// If TWEEK_BASE_DIR is not set, look up the path to this shared library
// and use it to provide a default setting for that environment variable.
if ( NULL == env_dir )
{
Dl_info info;
info.dli_fname = 0;
const int result =
#if defined(__GNUC__) && __GNUC_MAJOR__ < 4
dladdr((void*) &tweekLibraryInit, &info);
#else
dladdr(reinterpret_cast<void*>(&tweekLibraryInit), &info);
#endif
// NOTE: dladdr(3) really does return a non-zero value on success.
if ( 0 != result )
{
try
{
fs::path lib_file(info.dli_fname, fs::native);
lib_file = fs::system_complete(lib_file);
#if defined(VPR_OS_IRIX) && defined(_ABIN32)
const std::string bit_suffix("32");
#elif defined(VPR_OS_IRIX) && defined(_ABI64) || \
defined(VPR_OS_Linux) && defined(__x86_64__)
const std::string bit_suffix("64");
#else
const std::string bit_suffix("");
#endif
// Get the directory containing this shared library.
const fs::path lib_path = lib_file.branch_path();
// Start the search for the root of the Tweek installation in the
// parent of the directory containing this shared library.
base_dir = lib_path.branch_path();
// Use the lib subdirectory to figure out when we have found the
// root of the Tweek installation tree.
const fs::path lib_subdir(std::string("lib") + bit_suffix);
bool found(false);
while ( ! found && ! base_dir.empty() )
{
try
{
if ( ! fs::exists(base_dir / lib_subdir) )
{
base_dir = base_dir.branch_path();
}
else
{
found = true;
}
}
catch (fs::filesystem_error&)
{
base_dir = base_dir.branch_path();
}
}
if ( found )
{
setenv("TWEEK_BASE_DIR",
base_dir.native_directory_string().c_str(), 1);
}
}
catch (fs::filesystem_error& ex)
{
std::cerr << "Automatic assignment of TWEEK_BASE_DIR failed:\n"
<< ex.what() << std::endl;
}
}
}
else
{
try
{
base_dir = fs::path(env_dir, fs::native);
}
catch (fs::filesystem_error& ex)
{
std::cerr << "Invalid path set in TWEEK_BASE_DIR environment "
<< "variable:\n" << ex.what() << std::endl;
}
}
if ( ! base_dir.empty() )
{
// If base_dir were empty, this would result in data_dir being relative
// to the current working directory.
const fs::path data_dir = base_dir / TWEEK_SHARE_DIR;
// We use the overwrite value of 0 as a way around testing whether the
// environment variable is already set.
setenv("TWEEK_DATA_DIR", data_dir.native_directory_string().c_str(), 0);
}
}
inline boost::filesystem::path branch_path(boost::filesystem::path const& p)
{
return p.branch_path();
}
void
build_svg(
const fs::path & file,
const string & title,
const seq_t & seq,
float_t min_threshold,
match_result_vec_t & results,
size_t max_num_factors,
bool show_labels,
bool open,
match_result_vec_t * max_chain,
const std::string & notes,
float max_threshold )
{
XMLPlatformUtils::Initialize();
//find the maximum score
for (match_result_vec_t::const_iterator i = results.begin();
results.end() != i;
++i)
{
max_threshold = std::max( max_threshold, i->result.score );
}
SvgDomDocument doc(seq.size(), min_threshold, max_threshold, title, seq, show_labels);
//for each hit result
size_t idx = 0;
unsigned num_added = 0;
for (match_result_vec_t::iterator i = results.begin();
results.end() != i;
++i, ++idx)
{
i->number = idx;
BiobaseTablePssmEntry * pssm_entry = BiobaseDb::singleton().get_pssm_entry(i->link);
assert((size_t) i->result.position < seq.size());
if (min_threshold <= i->result.score)
{
const bool is_in_max_chain =
0 != max_chain
&&
max_chain->end() != std::find_if( max_chain->begin(), max_chain->end(), std::bind1st( detail::hit_is_same(), *i ) );
doc.add_result(
i->result,
*pssm_entry,
idx,
is_in_max_chain );
++num_added;
}
}
if (0 == num_added)
{
throw std::logic_error( "No hits above threshold" );
}
factor_scores_map_t factors;
find_factors_for_matches(results, factors);
//cout << factors.size() << " factors associated with these matches" << std::endl;
//for each factor - in score order
while (! factors.empty() && 0 != max_num_factors--)
{
//find the factor with the highest score
factor_scores_map_t::iterator best = factors.end();
for (factor_scores_map_t::iterator f = factors.begin();
factors.end() != f;
++f)
{
if (factors.end() == best || f->second.score > best->second.score)
{
best = f;
}
}
assert(best != factors.end());
Factor * factor = BiobaseDb::singleton().get_entry<FACTOR_DATA>(best->first);
if (0 != factor)
{
doc.add_factor(factor, best->second.hits);
}
factors.erase(best);
}
if( notes.size() )
{
doc.add_notes( notes );
}
add_tooltip_support(doc.doc, doc.doc_root);
dom_print(doc.doc, file._BOOST_FS_NATIVE().c_str());
//copy the script to the same directory
try
{
fs::path script(BioEnvironment::singleton().get_svg_script_file());
fs::path script_dest(file.branch_path());
script_dest /= script.leaf();
//only copy if destination older
if (! fs::exists(script_dest) || fs::last_write_time(script) > fs::last_write_time(script_dest))
{
//remove first if exists
if (fs::exists(script_dest))
{
fs::remove(script_dest);
}
fs::copy_file(script, script_dest);
}
}
catch (const std::exception & ex)
{
cerr << ex.what() << endl;
}
XMLPlatformUtils::Terminate();
if (open)
{
open_file(file);
}
}
void process_ipp_file(const fs::path& file, bool positive_test)
{
std::cout << "Info: Scanning file: " << file.string() << std::endl;
// our variables:
std::string file_text;
std::string macro_name;
std::string namespace_name;
fs::path positive_file;
fs::path negative_file;
// load the file into memory so we can scan it:
fs::ifstream ifs(file);
std::copy(std::istreambuf_iterator<char>(ifs), std::istreambuf_iterator<char>(), std::back_inserter(file_text));
ifs.close();
// scan for the macro name:
boost::regex macro_regex("//\\s*MACRO\\s*:\\s*(\\w+)");
boost::smatch macro_match;
if(boost::regex_search(file_text, macro_match, macro_regex))
{
macro_name = macro_match[1];
macro_list.insert(macro_name);
namespace_name = boost::regex_replace(file_text, macro_regex, "\\L$1", boost::format_first_only | boost::format_no_copy);
}
if(macro_name.empty())
{
std::cout << "Error: no macro definition found in " << file.string();
}
else
{
std::cout << "Info: Macroname: " << macro_name << std::endl;
}
// get the output filesnames:
boost::regex file_regex("boost_([^.]+)\\.ipp");
positive_file = file.branch_path() / boost::regex_replace(file.leaf().string(), file_regex, "$1_pass.cpp");
negative_file = file.branch_path() / boost::regex_replace(file.leaf().string(), file_regex, "$1_fail.cpp");
write_test_file(positive_file, macro_name, namespace_name, file.leaf().string(), positive_test, true);
write_test_file(negative_file, macro_name, namespace_name, file.leaf().string(), positive_test, false);
// always create config_test data,
// positive and negative tests go to separate streams, because for some
// reason some compilers choke unless we put them in a particular order...
std::ostream* pout = positive_test ? &config_test1a : &config_test1;
*pout << "#if";
if(!positive_test)
*pout << "n";
*pout << "def " << macro_name
<< "\n#include \"" << file.leaf().string() << "\"\n#else\nnamespace "
<< namespace_name << " = empty_boost;\n#endif\n";
config_test2 << " if(0 != " << namespace_name << "::test())\n"
" {\n"
" std::cerr << \"Failed test for " << macro_name << " at: \" << __FILE__ << \":\" << __LINE__ << std::endl;\n"
" ++error_count;\n"
" }\n";
// always generate the jamfile data:
jamfile << "test-suite \"" << macro_name << "\" : \n"
"[ run " << positive_file.leaf().string() << " <template>config_options ]\n"
"[ compile-fail " << negative_file.leaf().string() << " <template>config_options ] ;\n";
jamfile_v2 << "test-suite \"" << macro_name << "\" : \n"
"[ run ../" << positive_file.leaf().string() << " ]\n"
"[ compile-fail ../" << negative_file.leaf().string() << " ] ;\n";
// Generate data for the Build-checks test file:
build_config_test << "#ifdef TEST_" << macro_name << std::endl;
build_config_test << "# include \"../test/" << file.leaf().string() << "\"\n";
build_config_test << "namespace test = " << namespace_name << ";\n#endif\n";
// Generate data for the build-checks Jamfile:
static const boost::regex feature_regex("boost_(?:no|has)_(.*)");
std::string feature_name = boost::regex_replace(namespace_name, feature_regex, "\\1");
build_config_jamfile << "run-simple test_case.cpp : : : <define>TEST_" << macro_name << " : " << feature_name << " ;\n";
build_config_jamfile << "alias " << feature_name << " : " << feature_name << ".output ;\n";
build_config_jamfile << "explicit " << feature_name << " ;\n";
}
std::pair<const char*, bool> alert(const char* message_text,
const char* window_name,
const char* button_0_name,
const char* button_1_name,
const char* button_2_name,
const char* checkbox_name,
const boost::filesystem::path& icon_path,
std::size_t default_button_index,
std::size_t cancel_button_index)
{
if ((button_0_name || button_1_name || button_2_name) == false)
return std::make_pair<const char*, bool>(0, false);
std::stringstream sheet;
std::stringstream layout;
boost::filesystem::path icon_directory_path;
// The sheet for the alert dialog
sheet <<
"sheet alert_sheet\n"
"{\n"
"interface:\n"
" checkbox_value: false;\n"
"output:\n"
" result <== { checkbox_value: checkbox_value };\n"
"}\n"
;
// Start by filling out the header for the layout of the alert
layout <<
"layout alert_layout\n"
"{\n"
" view dialog(name: '" << window_name << "', placement: place_row, spacing: 10)\n"
" {\n";
if (icon_path != boost::filesystem::path())
{
icon_directory_path = icon_path.branch_path();
layout <<
" label_t(image: '" << icon_path.leaf() << "')\n"
;
}
layout <<
" column()\n"
" {\n"
" static_text(name:'" << message_text << "', horizontal: align_fill, characters: 25);\n"
;
// add the checkbox if we have a name for one
if (checkbox_name)
{
layout <<
"checkbox(name: '" << checkbox_name << "', bind: @checkbox_value);\n"
;
}
// add the button set
layout <<
"row(horizontal: align_right)\n"
"{\n"
;
// add the buttons in *reverse* order, so the first is rightmost
append_button_to_layout(layout, button_2_name, 2, default_button_index == 2, cancel_button_index == 2);
append_button_to_layout(layout, button_1_name, 1, default_button_index == 1, cancel_button_index == 1);
append_button_to_layout(layout, button_0_name, 0, default_button_index == 0, cancel_button_index == 0);
// close out the rest of the layout
layout <<
" }\n" // row
" }\n" // column
" }\n" // dialog
"}\n" // layout
;
// finally set up the params for the modal dialog interface call
dialog_result_t result(handle_dialog(dictionary_t(),
dictionary_t(),
dictionary_t(),
dialog_display_s,
layout,
sheet,
&always_break,
icon_directory_path));
bool is_checked(get_value(result.command_m, static_name_t("checkbox_value")).cast<bool>());
// NOTE (fbrereto) : Here is why we require the name of the action to be "bN", where N is the index
std::size_t index(std::atoi(&result.terminating_action_m.c_str()[1]));
if (index == 0)
return std::make_pair(button_0_name, is_checked);
else if (index == 1)
return std::make_pair(button_1_name, is_checked);
return std::make_pair(button_2_name, is_checked);
}
void drillDown( boost::filesystem::path root,
bool use_db,
bool use_coll,
bool oplogReplayLimit,
bool top_level=false) {
bool json_metadata = false;
LOG(2) << "drillDown: " << root.string() << endl;
// skip hidden files and directories
if (root.leaf()[0] == '.' && root.leaf() != ".")
return;
if ( is_directory( root ) ) {
boost::filesystem::directory_iterator end;
boost::filesystem::directory_iterator i(root);
boost::filesystem::path indexes;
while ( i != end ) {
boost::filesystem::path p = *i;
i++;
if (use_db) {
if (boost::filesystem::is_directory(p)) {
error() << "ERROR: root directory must be a dump of a single database" << endl;
error() << " when specifying a db name with --db" << endl;
printHelp(cout);
return;
}
}
if (use_coll) {
if (boost::filesystem::is_directory(p) || i != end) {
error() << "ERROR: root directory must be a dump of a single collection" << endl;
error() << " when specifying a collection name with --collection" << endl;
printHelp(cout);
return;
}
}
// don't insert oplog
if (top_level && !use_db && p.leaf() == "oplog.bson")
continue;
if ( p.leaf() == "system.indexes.bson" ) {
indexes = p;
} else {
drillDown(p, use_db, use_coll, oplogReplayLimit);
}
}
if (!indexes.empty() && !json_metadata) {
drillDown(indexes, use_db, use_coll, oplogReplayLimit);
}
return;
}
if ( endsWith( root.string().c_str() , ".metadata.json" ) ) {
// Metadata files are handled when the corresponding .bson file is handled
return;
}
if ( ! ( endsWith( root.string().c_str() , ".bson" ) ||
endsWith( root.string().c_str() , ".bin" ) ) ) {
error() << "don't know what to do with file [" << root.string() << "]" << endl;
return;
}
log() << root.string() << endl;
if ( root.leaf() == "system.profile.bson" ) {
log() << "\t skipping" << endl;
return;
}
string ns;
if (use_db) {
ns += _db;
}
else {
string dir = root.branch_path().string();
if ( dir.find( "/" ) == string::npos )
ns += dir;
else
ns += dir.substr( dir.find_last_of( "/" ) + 1 );
if ( ns.size() == 0 )
ns = "test";
}
verify( ns.size() );
string oldCollName = root.leaf(); // Name of the collection that was dumped from
oldCollName = oldCollName.substr( 0 , oldCollName.find_last_of( "." ) );
if (use_coll) {
ns += "." + _coll;
}
else {
ns += "." + oldCollName;
}
if (oplogReplayLimit) {
error() << "The oplogLimit option cannot be used if "
<< "normal databases/collections exist in the dump directory."
<< endl;
exit(EXIT_FAILURE);
}
log() << "\tgoing into namespace [" << ns << "]" << endl;
if ( _drop ) {
if (root.leaf() != "system.users.bson" ) {
log() << "\t dropping" << endl;
conn().dropCollection( ns );
} else {
// Create map of the users currently in the DB
BSONObj fields = BSON("user" << 1);
scoped_ptr<DBClientCursor> cursor(conn().query(ns, Query(), 0, 0, &fields));
while (cursor->more()) {
BSONObj user = cursor->next();
_users.insert(user["user"].String());
}
}
}
BSONObj metadataObject;
if (_restoreOptions || _restoreIndexes) {
boost::filesystem::path metadataFile = (root.branch_path() / (oldCollName + ".metadata.json"));
if (!boost::filesystem::exists(metadataFile.string())) {
// This is fine because dumps from before 2.1 won't have a metadata file, just print a warning.
// System collections shouldn't have metadata so don't warn if that file is missing.
if (!startsWith(metadataFile.leaf(), "system.")) {
log() << metadataFile.string() << " not found. Skipping." << endl;
}
} else {
metadataObject = parseMetadataFile(metadataFile.string());
}
}
_curns = ns.c_str();
_curdb = NamespaceString(_curns).db;
_curcoll = NamespaceString(_curns).coll;
// If drop is not used, warn if the collection exists.
if (!_drop) {
scoped_ptr<DBClientCursor> cursor(conn().query(_curdb + ".system.namespaces",
Query(BSON("name" << ns))));
if (cursor->more()) {
// collection already exists show warning
warning() << "Restoring to " << ns << " without dropping. Restored data "
"will be inserted without raising errors; check your server log"
<< endl;
}
}
if (_restoreOptions && metadataObject.hasField("options")) {
// Try to create collection with given options
createCollectionWithOptions(metadataObject["options"].Obj());
}
processFile( root );
if (_drop && root.leaf() == "system.users.bson") {
// Delete any users that used to exist but weren't in the dump file
for (set<string>::iterator it = _users.begin(); it != _users.end(); ++it) {
BSONObj userMatch = BSON("user" << *it);
conn().remove(ns, Query(userMatch));
}
_users.clear();
}
if (_restoreIndexes && metadataObject.hasField("indexes")) {
vector<BSONElement> indexes = metadataObject["indexes"].Array();
for (vector<BSONElement>::iterator it = indexes.begin(); it != indexes.end(); ++it) {
createIndex((*it).Obj(), false);
}
}
}
int main(int argc, char** argv)
{
Waifu2x::init_liblary(argc, argv);
// Caffeのエラーでないログを保存しないようにする
google::SetLogDestination(google::INFO, "");
google::SetLogDestination(google::WARNING, "");
// Caffeのエラーログを「error_log_〜」に出力
google::SetLogDestination(google::ERROR, "error_log_");
google::SetLogDestination(google::FATAL, "error_log_");
// definition of command line arguments
TCLAP::CmdLine cmd("waifu2x reimplementation using Caffe", ' ', "1.0.0");
TCLAP::ValueArg<std::string> cmdInputFile("i", "input_path",
"path to input image file", true, "",
"string", cmd);
TCLAP::ValueArg<std::string> cmdOutputFile("o", "output_path",
"path to output image file (when input_path is folder, output_path must be folder)", false,
"(auto)", "string", cmd);
TCLAP::ValueArg<std::string> cmdInputFileExt("l", "input_extention_list",
"extention to input image file when input_path is folder", false, "png:jpg:jpeg:tif:tiff:bmp:tga",
"string", cmd);
TCLAP::ValueArg<std::string> cmdOutputFileExt("e", "output_extention",
"extention to output image file when output_path is (auto) or input_path is folder", false,
"png", "string", cmd);
std::vector<std::string> cmdModeConstraintV;
cmdModeConstraintV.push_back("noise");
cmdModeConstraintV.push_back("scale");
cmdModeConstraintV.push_back("noise_scale");
cmdModeConstraintV.push_back("auto_scale");
TCLAP::ValuesConstraint<std::string> cmdModeConstraint(cmdModeConstraintV);
TCLAP::ValueArg<std::string> cmdMode("m", "mode", "image processing mode",
false, "noise_scale", &cmdModeConstraint, cmd);
std::vector<int> cmdNRLConstraintV;
cmdNRLConstraintV.push_back(1);
cmdNRLConstraintV.push_back(2);
cmdNRLConstraintV.push_back(3);
TCLAP::ValuesConstraint<int> cmdNRLConstraint(cmdNRLConstraintV);
TCLAP::ValueArg<int> cmdNRLevel("n", "noise_level", "noise reduction level",
false, 1, &cmdNRLConstraint, cmd);
TCLAP::ValueArg<double> cmdScaleRatio("s", "scale_ratio",
"custom scale ratio", false, 2.0, "double", cmd);
TCLAP::ValueArg<double> cmdScaleWidth("w", "scale_width",
"custom scale width", false, 0, "double", cmd);
TCLAP::ValueArg<double> cmdScaleHeight("h", "scale_height",
"custom scale height", false, 0, "double", cmd);
TCLAP::ValueArg<std::string> cmdModelPath("", "model_dir",
"path to custom model directory (don't append last / )", false,
"models/anime_style_art_rgb", "string", cmd);
std::vector<std::string> cmdProcessConstraintV;
cmdProcessConstraintV.push_back("cpu");
cmdProcessConstraintV.push_back("gpu");
cmdProcessConstraintV.push_back("cudnn");
TCLAP::ValuesConstraint<std::string> cmdProcessConstraint(cmdProcessConstraintV);
TCLAP::ValueArg<std::string> cmdProcess("p", "process", "process mode",
false, "gpu", &cmdProcessConstraint, cmd);
TCLAP::ValueArg<int> cmdOutputQuality("q", "output_quality",
"output image quality", false,
-1, "int", cmd);
TCLAP::ValueArg<int> cmdOutputDepth("d", "output_depth",
"output image chaneel depth bit", false,
8, "int", cmd);
TCLAP::ValueArg<int> cmdCropSizeFile("c", "crop_size",
"input image split size", false,
128, "int", cmd);
TCLAP::ValueArg<int> cmdCropWidth("", "crop_w",
"input image split size(width)", false,
128, "int", cmd);
TCLAP::ValueArg<int> cmdCropHeight("", "crop_h",
"input image split size(height)", false,
128, "int", cmd);
TCLAP::ValueArg<int> cmdBatchSizeFile("b", "batch_size",
"input batch size", false,
1, "int", cmd);
TCLAP::ValueArg<int> cmdGPUNoFile("", "gpu",
"gpu device no", false,
0, "int", cmd);
std::vector<int> cmdTTAConstraintV;
cmdTTAConstraintV.push_back(0);
cmdTTAConstraintV.push_back(1);
TCLAP::ValuesConstraint<int> cmdTTAConstraint(cmdTTAConstraintV);
TCLAP::ValueArg<int> cmdTTALevel("t", "tta", "8x slower and slightly high quality",
false, 0, &cmdTTAConstraint, cmd);
// definition of command line argument : end
TCLAP::Arg::enableIgnoreMismatched();
// parse command line arguments
try
{
cmd.parse(argc, argv);
}
catch (std::exception &e)
{
printf("エラー: %s\n", e.what());
return 1;
}
boost::optional<double> ScaleRatio;
boost::optional<int> ScaleWidth;
boost::optional<int> ScaleHeight;
int valid_num = 0;
if (cmdScaleWidth.getValue() > 0)
valid_num++;
if (cmdScaleHeight.getValue() > 0)
valid_num++;
if (valid_num > 1)
{
printf("エラー: scale_widthとscale_heightは同時に指定できません\n");
return 1;
}
int crop_w = cmdCropSizeFile.getValue();
int crop_h = cmdCropSizeFile.getValue();
if (cmdCropWidth.isSet())
crop_w = cmdCropWidth.getValue();
if (cmdCropHeight.isSet())
crop_h = cmdCropHeight.getValue();
if (cmdScaleWidth.getValue() > 0)
ScaleWidth = cmdScaleWidth.getValue();
else if (cmdScaleHeight.getValue() > 0)
ScaleHeight = cmdScaleHeight.getValue();
else
ScaleRatio = cmdScaleRatio.getValue();
const boost::filesystem::path input_path(boost::filesystem::absolute((cmdInputFile.getValue())));
std::string outputExt = cmdOutputFileExt.getValue();
if (outputExt.length() > 0 && outputExt[0] != '.')
outputExt = "." + outputExt;
const std::string ModelName = Waifu2x::GetModelName(cmdModelPath.getValue());
const bool use_tta = cmdTTALevel.getValue() == 1;
std::vector<std::pair<std::string, std::string>> file_paths;
if (boost::filesystem::is_directory(input_path)) // input_pathがフォルダならそのディレクトリ以下の画像ファイルを一括変換
{
boost::filesystem::path output_path;
if (cmdOutputFile.getValue() == "(auto)")
{
// 「test」なら「test_noise_scale(Level1)(x2.000000)」みたいな感じにする
std::string addstr("(");
addstr += ModelName;
addstr += ")";
const std::string &mode = cmdMode.getValue();
addstr += "(" + mode + ")";
if (mode.find("noise") != mode.npos || mode.find("auto_scale") != mode.npos)
addstr += "(Level" + std::to_string(cmdNRLevel.getValue()) + ")";
if (use_tta)
addstr += "(tta)";
if (mode.find("scale") != mode.npos)
{
if(ScaleRatio)
addstr += "(x" + std::to_string(*ScaleRatio) + ")";
else if (ScaleWidth)
addstr += "(width " + std::to_string(*ScaleWidth) + ")";
else
addstr += "(height " + std::to_string(*ScaleHeight) + ")";
}
if (cmdOutputDepth.getValue() != 8)
addstr += "(" + std::to_string(cmdOutputDepth.getValue()) + "bit)";
output_path = input_path.branch_path() / (input_path.stem().string() + addstr);
}
else
output_path = cmdOutputFile.getValue();
output_path = boost::filesystem::absolute(output_path);
if (!boost::filesystem::exists(output_path))
{
if (!boost::filesystem::create_directory(output_path))
{
printf("エラー: 出力フォルダ「%s」の作成に失敗しました\n", output_path.string().c_str());
return 1;
}
}
std::vector<std::string> extList;
{
// input_extention_listを文字列の配列にする
typedef boost::char_separator<char> char_separator;
typedef boost::tokenizer<char_separator> tokenizer;
char_separator sep(":", "", boost::drop_empty_tokens);
tokenizer tokens(cmdInputFileExt.getValue(), sep);
for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter)
{
std::string ext(*tok_iter);
std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
extList.push_back("." + ext);
}
}
// 変換する画像の入力、出力パスを取得
const auto func = [&extList, &input_path, &output_path, &outputExt, &file_paths](const boost::filesystem::path &path)
{
BOOST_FOREACH(const boost::filesystem::path& p, std::make_pair(boost::filesystem::recursive_directory_iterator(path),
boost::filesystem::recursive_directory_iterator()))
{
if (boost::filesystem::is_directory(p))
{
const auto out_relative = relativePath(p, input_path);
const auto out_absolute = output_path / out_relative;
if (!boost::filesystem::exists(out_absolute))
{
if (!boost::filesystem::create_directory(out_absolute))
{
printf("エラー: 出力フォルダ「%s」の作成に失敗しました\n", out_absolute.string().c_str());
return false;
}
}
}
else
{
std::string ext(p.extension().string());
std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
if (std::find(extList.begin(), extList.end(), ext) != extList.end())
{
const auto out_relative = relativePath(p, input_path);
const auto out_absolute = output_path / out_relative;
const auto out = (out_absolute.branch_path() / out_absolute.stem()).string() + outputExt;
file_paths.emplace_back(p.string(), out);
}
}
}
return true;
};
if (!func(input_path))
return 1;
}
else
{
int main(int argc, char** argv)
{
// definition of command line arguments
TCLAP::CmdLine cmd("waifu2x reimplementation using Caffe", ' ', "1.0.0");
TCLAP::ValueArg<std::string> cmdInputFile("i", "input_path",
"path to input image file", true, "",
"string", cmd);
TCLAP::ValueArg<std::string> cmdOutputFile("o", "output_path",
"path to output image file (when input_path is folder, output_path must be folder)", false,
"(auto)", "string", cmd);
TCLAP::ValueArg<std::string> cmdInputFileExt("l", "input_extention_list",
"extention to input image file when input_path is folder", false, "png:jpg:jpeg:tif:tiff:bmp",
"string", cmd);
TCLAP::ValueArg<std::string> cmdOutputFileExt("e", "output_extention",
"extention to output image file when output_path is (auto) or input_path is folder", false,
"png", "string", cmd);
std::vector<std::string> cmdModeConstraintV;
cmdModeConstraintV.push_back("noise");
cmdModeConstraintV.push_back("scale");
cmdModeConstraintV.push_back("noise_scale");
cmdModeConstraintV.push_back("auto_scale");
TCLAP::ValuesConstraint<std::string> cmdModeConstraint(cmdModeConstraintV);
TCLAP::ValueArg<std::string> cmdMode("m", "mode", "image processing mode",
false, "noise_scale", &cmdModeConstraint, cmd);
std::vector<int> cmdNRLConstraintV;
cmdNRLConstraintV.push_back(1);
cmdNRLConstraintV.push_back(2);
TCLAP::ValuesConstraint<int> cmdNRLConstraint(cmdNRLConstraintV);
TCLAP::ValueArg<int> cmdNRLevel("n", "noise_level", "noise reduction level",
false, 1, &cmdNRLConstraint, cmd);
TCLAP::ValueArg<double> cmdScaleRatio("s", "scale_ratio",
"custom scale ratio", false, 2.0, "double", cmd);
TCLAP::ValueArg<std::string> cmdModelPath("", "model_dir",
"path to custom model directory (don't append last / )", false,
"models", "string", cmd);
std::vector<std::string> cmdProcessConstraintV;
cmdProcessConstraintV.push_back("cpu");
cmdProcessConstraintV.push_back("gpu");
cmdProcessConstraintV.push_back("cudnn");
TCLAP::ValuesConstraint<std::string> cmdProcessConstraint(cmdProcessConstraintV);
TCLAP::ValueArg<std::string> cmdProcess("p", "process", "process mode",
false, "gpu", &cmdProcessConstraint, cmd);
TCLAP::ValueArg<int> cmdCropSizeFile("c", "crop_size",
"input image split size", false,
128, "int", cmd);
TCLAP::ValueArg<int> cmdBatchSizeFile("b", "batch_size",
"input batch size", false,
1, "int", cmd);
// definition of command line argument : end
TCLAP::Arg::enableIgnoreMismatched();
// parse command line arguments
try
{
cmd.parse(argc, argv);
}
catch (std::exception &e)
{
printf("エラー: %s\n", e.what());
return 1;
}
const boost::filesystem::path input_path(boost::filesystem::absolute((cmdInputFile.getValue())));
std::string outputExt = cmdOutputFileExt.getValue();
if (outputExt.length() > 0 && outputExt[0] != '.')
outputExt = "." + outputExt;
std::vector<std::pair<std::string, std::string>> file_paths;
if (boost::filesystem::is_directory(input_path)) // input_pathがフォルダならそのディレクトリ以下の画像ファイルを一括変換
{
boost::filesystem::path output_path;
if (cmdOutputFile.getValue() == "(auto)")
{
// 「test」なら「test_noise_scale(Level1)(x2.000000)」みたいな感じにする
std::string addstr("_" + cmdMode.getValue());
const std::string &mode = cmdMode.getValue();
if (mode.find("noise") != mode.npos || mode.find("auto_scale") != mode.npos)
addstr += "(Level" + std::to_string(cmdNRLevel.getValue()) + ")";
if (mode.find("scale") != mode.npos)
addstr += "(x" + std::to_string(cmdScaleRatio.getValue()) + ")";
output_path = input_path.branch_path() / (input_path.stem().string() + addstr);
}
else
output_path = cmdOutputFile.getValue();
output_path = boost::filesystem::absolute(output_path);
if (!boost::filesystem::exists(output_path))
{
if (!boost::filesystem::create_directory(output_path))
{
printf("エラー: 出力フォルダ「%s」の作成に失敗しました\n", output_path.string().c_str());
return 1;
}
}
std::vector<std::string> extList;
{
// input_extention_listを文字列の配列にする
typedef boost::char_separator<char> char_separator;
typedef boost::tokenizer<char_separator> tokenizer;
char_separator sep(":", "", boost::drop_empty_tokens);
tokenizer tokens(cmdInputFileExt.getValue(), sep);
for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter)
extList.push_back("." + *tok_iter);
}
// 変換する画像の入力、出力パスを取得
const auto func = [&extList, &input_path, &output_path, &outputExt, &file_paths](const boost::filesystem::path &path)
{
BOOST_FOREACH(const boost::filesystem::path& p, std::make_pair(boost::filesystem::recursive_directory_iterator(path),
boost::filesystem::recursive_directory_iterator()))
{
if (boost::filesystem::is_directory(p))
{
const auto out_relative = relativePath(p, input_path);
const auto out_absolute = output_path / out_relative;
if (!boost::filesystem::exists(out_absolute))
{
if (!boost::filesystem::create_directory(out_absolute))
{
printf("エラー: 出力フォルダ「%s」の作成に失敗しました\n", out_absolute.string().c_str());
return false;
}
}
}
else if (std::find(extList.begin(), extList.end(), p.extension().string()) != extList.end())
{
const auto out_relative = relativePath(p, input_path);
const auto out_absolute = output_path / out_relative;
const auto out = (out_absolute.branch_path() / out_absolute.stem()).string() + outputExt;
file_paths.emplace_back(p.string(), out);
}
}
return true;
};
if (!func(input_path))
return 1;
}
else
{
/**
* Non-Windows shared library constructor. This ensures that the environment
* variable \c GADGET_BASE_DIR is set as soon as this shared library is loaded.
* If it is not set, then it sets it based on an assumption about the
* structure of a Gadgeteer installation. More specifically, an assumption is
* made that this shared library lives in the \c lib subdirectory of the
* Gadgeteer installation. Therefore, the root of the Gadgeteer installation
* is the parent of the directory containing this shared library.
*/
extern "C" void __attribute ((constructor)) gadgetLibraryInit()
{
fs::path base_dir;
const char* env_dir = std::getenv("GADGET_BASE_DIR");
// If GADGET_BASE_DIR is not set, look up the path to this shared library
// and use it to provide a default setting for that environment variable.
if ( NULL == env_dir )
{
Dl_info info;
info.dli_fname = 0;
const int result =
#if defined(__GNUC__) && __GNUC_MAJOR__ < 4
dladdr((void*) &gadgetLibraryInit, &info);
#else
dladdr(reinterpret_cast<void*>(&gadgetLibraryInit), &info);
#endif
// NOTE: dladdr(3) really does return a non-zero value on success.
if ( 0 != result )
{
try
{
fs::path lib_file(
#if BOOST_VERSION >= 104600 && BOOST_FILESYSTEM_VERSION == 3
info.dli_fname
#else
info.dli_fname, fs::native
#endif
);
lib_file = fs::system_complete(lib_file);
#if defined(GADGET_LIBDIR_NAME)
const std::string lib_dir_name(GADGET_LIBDIR_NAME);
#else
const std::string lib_dir_name("lib");
#endif
// Get the directory containing this shared library.
const fs::path lib_path = lib_file.branch_path();
// Start the search for the root of the Gadgeteer installation in
// the parent of the directory containing this shared library.
base_dir = lib_path.branch_path();
// Use the lib subdirectory to figure out when we have found the
// root of the Gadgeteer installation tree.
const fs::path lib_subdir(lib_dir_name);
bool found(false);
while ( ! found && ! base_dir.empty() )
{
try
{
if ( ! fs::exists(base_dir / lib_subdir) )
{
base_dir = base_dir.branch_path();
}
else
{
found = true;
}
}
catch (fs::filesystem_error&)
{
base_dir = base_dir.branch_path();
}
}
if ( found )
{
setenv("GADGET_BASE_DIR",
#if BOOST_VERSION >= 104600 && BOOST_FILESYSTEM_VERSION == 3
base_dir.native().c_str(),
#else
base_dir.native_directory_string().c_str(),
#endif
1);
}
}
catch (fs::filesystem_error& ex)
{
std::cerr << "Automatic assignment of GADGET_BASE_DIR failed:\n"
<< ex.what() << std::endl;
}
}
}
else
{
try
{
base_dir =
#if BOOST_VERSION >= 104600 && BOOST_FILESYSTEM_VERSION == 3
fs::path(env_dir);
#else
fs::path(env_dir, fs::native);
#endif
}
catch (fs::filesystem_error& ex)
{
std::cerr << "Invalid path set in GADGET_BASE_DIR environment "
<< "variable:\n" << ex.what() << std::endl;
}
}
if ( ! base_dir.empty() )
{
// If base_dir were empty, this would result in data_dir being relative
// to the current working directory.
const fs::path data_dir = base_dir / GADGET_SHARE_DIR;
// We use the overwrite value of 0 as a way around testing whether the
// environment variable is already set.
setenv("GADGET_DATA_DIR",
#if BOOST_VERSION >= 104600 && BOOST_FILESYSTEM_VERSION == 3
data_dir.native().c_str(),
#else
data_dir.native_directory_string().c_str(),
#endif
0);
}
}
bool JSBase::internalInitialize(
const boost::filesystem::path& scriptFile, const std::string& functionName,
void(*extensionGlobals)(OSS_HANDLE) )
{
if (!boost::filesystem::exists(scriptFile))
{
OSS_LOG_ERROR("Google V8 is unable to locate file " << scriptFile);
return false;
}
OSS_LOG_INFO("Google V8 JSBase::internalInitialize INVOKED");
// Create a handle scope to hold the temporary references.
v8::HandleScope handle_scope;
v8::Persistent<v8::ObjectTemplate>* oldGlobalTemplate = static_cast<v8::Persistent<v8::ObjectTemplate>*>(_globalTemplate);
v8::Persistent<v8::ObjectTemplate>* oldRequestTemplate = static_cast<v8::Persistent<v8::ObjectTemplate>*>(_requestTemplate);
v8::Persistent<v8::Function>* oldProcessFunc = static_cast<v8::Persistent<v8::Function>*>(_processFunc);
v8::Persistent<v8::Context>* oldContext = static_cast<v8::Persistent<v8::Context>*>(_context);
if (oldContext)
{
(*oldContext)->DetachGlobal();
oldContext->Dispose();
delete static_cast<v8::Persistent<v8::Context>*>(oldContext);
_context = 0;
}
if (oldGlobalTemplate)
{
oldGlobalTemplate->Dispose();
delete static_cast<v8::Persistent<v8::ObjectTemplate>*>(oldGlobalTemplate);
_globalTemplate = 0;
}
if (oldRequestTemplate)
{
oldRequestTemplate->Dispose();
delete static_cast<v8::Persistent<v8::ObjectTemplate>*>(oldRequestTemplate);
_requestTemplate = 0;
}
if (oldProcessFunc)
{
oldProcessFunc->Dispose();
delete static_cast<v8::Persistent<v8::Function>*>(oldProcessFunc);
_processFunc = 0;
}
v8::Persistent<v8::Context>* context_ = new v8::Persistent<v8::Context>();
v8::Persistent<v8::Function>* processFunc_ = new v8::Persistent<v8::Function>();
v8::Persistent<v8::ObjectTemplate>* requestTemplate_ = new v8::Persistent<v8::ObjectTemplate>;
v8::Persistent<v8::ObjectTemplate>* globalTemplate_ = new v8::Persistent<v8::ObjectTemplate>;
_context = context_;
_processFunc = processFunc_;
_requestTemplate = requestTemplate_;
_globalTemplate = globalTemplate_;
_functionName = functionName;
_script = scriptFile;
_extensionGlobals = extensionGlobals;
if (!_hasSetErrorCB)
{
if (!_hasSetErrorCB)
v8::V8::AddMessageListener(V8ErrorMessageCallback);
}
OSS_LOG_INFO("Google V8 is loading context for " << _script);
// Create a template for the global object where we set the
// built-in global functions.
//v8::Handle<v8::ObjectTemplate> global = v8::ObjectTemplate::New();
v8::Handle<v8::ObjectTemplate> global = v8::ObjectTemplate::New();
*(static_cast<v8::Persistent<v8::ObjectTemplate>*>(_globalTemplate)) = v8::Persistent<v8::ObjectTemplate>::New(global);
global->Set(v8::String::New("log_info"), v8::FunctionTemplate::New(log_info_callback));
global->Set(v8::String::New("log_debug"), v8::FunctionTemplate::New(log_debug_callback));
global->Set(v8::String::New("log_error"), v8::FunctionTemplate::New(log_error_callback));
//
// Initialize subclass global functions
//
OSS_LOG_INFO("Google V8 is initializing global exports for " << _script);
initGlobalFuncs(_globalTemplate);
//
// Initialize extension funcs
//
if (extensionGlobals)
extensionGlobals(_globalTemplate);
// Each processor gets its own context so different processors
// don't affect each other (ignore the first three lines).
v8::Handle<v8::Context> context = v8::Context::New(0, global);
// Store the context in the processor object in a persistent handle,
// since we want the reference to remain after we return from this
// method.
*(static_cast<v8::Persistent<v8::Context>*>(_context)) = v8::Persistent<v8::Context>::New(context);
// Enter the new context so all the following operations take place
// within it.
v8::Context::Scope context_scope(context);
OSS_LOG_INFO("Google V8 context for " << _script << " CREATED");
//
// We're just about to compile the script; set up an error handler to
// catch any exceptions the script might throw.
v8::TryCatch try_catch;
try_catch.SetVerbose(true);
//
// Compile global helpers
//
boost::filesystem::path helpers;
if (!_globalScriptsDirectory.empty())
helpers = boost::filesystem::path(_globalScriptsDirectory);
else
helpers = operator/(scriptFile.branch_path(), "global.detail");
if (boost::filesystem::exists(helpers))
{
try
{
//
// Compile it!
//
v8::Handle<v8::String> helperScript;
helperScript = read_directory(helpers);
if (helperScript.IsEmpty())
{
OSS_LOG_INFO("Google V8 global.detail for " << _script << " failed to compile.");
// The script failed to compile; bail out.
return false;
}
OSS_LOG_INFO("Google V8 is compiling global.detail for " << _script);
v8::Handle<v8::Script> compiledHelper = v8::Script::Compile(helperScript);
if (compiledHelper.IsEmpty())
{
reportException(try_catch, true);
return false;
}
OSS_LOG_INFO("Google V8 is running global.detail for " << _script);
// Run the script!
v8::Handle<v8::Value> result = compiledHelper->Run();
if (result.IsEmpty())
{
// The TryCatch above is still in effect and will have caught the error.
reportException(try_catch, true);
return false;
}
OSS_LOG_INFO("Google V8 global.detail for " << _script << " EXECUTED");
}
catch(OSS::Exception e)
{
std::ostringstream logMsg;
logMsg << "Filesystem error while compiling script global helpers - " << e.message();
OSS::log_warning(logMsg.str());
}
}
else
{
OSS_LOG_INFO("Google V8 context for " << _script << " is unable to load global exports from " << helpers.string());
}
//
// Compile the helpers
//
if (!_helperScriptsDirectory.empty())
helpers = boost::filesystem::path(_helperScriptsDirectory);
else
helpers = OSS::boost_path(_script) + ".detail";
if (boost::filesystem::exists(helpers))
{
//
// This script has a heper directory
//
try
{
boost::filesystem::directory_iterator end_itr; // default construction yields past-the-end
for (boost::filesystem::directory_iterator itr(helpers); itr != end_itr; ++itr)
{
if (boost::filesystem::is_directory(itr->status()))
{
continue;
}
else
{
std::string fileName = OSS::boost_file_name(itr->path());
boost::filesystem::path currentFile = itr->path();
if (boost::filesystem::is_regular(currentFile))
{
if (OSS::string_ends_with(fileName, ".js"))
{
//
// Compile it!
//
OSS_LOG_INFO("Google V8 is compiling helper script " << currentFile);
v8::Handle<v8::String> helperScript;
helperScript = read_file(OSS::boost_path(currentFile));
if (helperScript.IsEmpty())
{
reportException(try_catch, true);
return false;
}
v8::Handle<v8::Script> compiledHelper = v8::Script::Compile(helperScript);
if (compiledHelper.IsEmpty())
{
reportException(try_catch, true);
return false;
}
// Run the script!
v8::Handle<v8::Value> result = compiledHelper->Run();
if (result.IsEmpty())
{
// The TryCatch above is still in effect and will have caught the error.
reportException(try_catch, true);
return false;
}
}
}
}
}
}
catch(OSS::Exception e)
{
std::ostringstream logMsg;
logMsg << "Filesystem error while compiling script helpers - " << e.message();
OSS::log_warning(logMsg.str());
}
}
//
// Compile the main script script
//
OSS_LOG_INFO("Google V8 is compiling main script " << _script);
v8::Handle<v8::String> script;
script = read_file(OSS::boost_path(_script));
v8::Handle<v8::Script> compiled_script = v8::Script::Compile(script);
if (compiled_script.IsEmpty())
{
reportException(try_catch, true);
return false;
}
OSS_LOG_INFO("Google V8 is running main script " << _script);
// Run the script!
v8::Handle<v8::Value> result = compiled_script->Run();
if (result.IsEmpty())
{
// The TryCatch above is still in effect and will have caught the error.
reportException(try_catch, true);
return false;
}
// The script compiled and ran correctly. Now we fetch out the
// Process function from the global object.
v8::Handle<v8::String> process_name = v8::String::New(functionName.c_str());
v8::Handle<v8::Value> process_val = context->Global()->Get(process_name);
// If there is no Process function, or if it is not a function,
// bail out
if (!process_val->IsFunction())
{
OSS_LOG_ERROR("Google V8 is unable to load function " << functionName);
return false;
}
// It is a function; cast it to a Function
v8::Handle<v8::Function> process_fun = v8::Handle<v8::Function>::Cast(process_val);
// Store the function in a Persistent handle, since we also want
// that to remain after this call returns
*(static_cast<v8::Persistent<v8::Function>*>(_processFunc)) = v8::Persistent<v8::Function>::New(process_fun);
// all went well. request the template creation as the final step
v8::Handle<v8::ObjectTemplate> objectTemplate = v8::ObjectTemplate::New();
objectTemplate->SetInternalFieldCount(1);
*(static_cast<v8::Persistent<v8::ObjectTemplate>*>(_requestTemplate)) = v8::Persistent<v8::ObjectTemplate>::New(objectTemplate);
_isInitialized = true;
return _isInitialized;
}
void drillDown( boost::filesystem::path root, bool use_db, bool use_coll, bool top_level=false ) {
LOG(2) << "drillDown: " << root.string() << endl;
// skip hidden files and directories
if (root.leaf().string()[0] == '.' && root.leaf().string() != ".")
return;
if ( is_directory( root ) ) {
boost::filesystem::directory_iterator end;
boost::filesystem::directory_iterator i(root);
while ( i != end ) {
boost::filesystem::path p = *i;
i++;
if (use_db) {
if (boost::filesystem::is_directory(p)) {
error() << "ERROR: root directory must be a dump of a single database" << endl;
error() << " when specifying a db name with --db" << endl;
printHelp(cout);
return;
}
}
if (use_coll) {
if (boost::filesystem::is_directory(p) || i != end) {
error() << "ERROR: root directory must be a dump of a single collection" << endl;
error() << " when specifying a collection name with --collection" << endl;
printHelp(cout);
return;
}
}
// don't insert oplog
if (top_level && !use_db && p.leaf() == "oplog.bson")
continue;
// Only restore indexes from a corresponding .metadata.json file.
if ( p.leaf() != "system.indexes.bson" ) {
drillDown(p, use_db, use_coll);
}
}
return;
}
if ( endsWith( root.string().c_str() , ".metadata.json" ) ) {
// Metadata files are handled when the corresponding .bson file is handled
return;
}
if ( ! ( endsWith( root.string().c_str() , ".bson" ) ||
endsWith( root.string().c_str() , ".bin" ) ) ) {
error() << "don't know what to do with file [" << root.string() << "]" << endl;
return;
}
log() << root.string() << endl;
if ( root.leaf() == "system.profile.bson" ) {
log() << "\t skipping" << endl;
return;
}
string ns;
if (use_db) {
ns += _db;
}
else {
ns = root.parent_path().filename().string();
if (ns.empty())
ns = "test";
}
verify( ns.size() );
string oldCollName = root.leaf().string(); // Name of the collection that was dumped from
oldCollName = oldCollName.substr( 0 , oldCollName.find_last_of( "." ) );
if (use_coll) {
ns += "." + _coll;
}
else {
ns += "." + oldCollName;
}
log() << "\tgoing into namespace [" << ns << "]" << endl;
if ( _drop ) {
if (root.leaf() != "system.users.bson" ) {
log() << "\t dropping" << endl;
conn().dropCollection( ns );
} else {
// Create map of the users currently in the DB
BSONObj fields = BSON("user" << 1);
scoped_ptr<DBClientCursor> cursor(conn().query(ns, Query(), 0, 0, &fields));
while (cursor->more()) {
BSONObj user = cursor->next();
_users.insert(user["user"].String());
}
}
}
BSONObj metadataObject;
if (_restoreOptions || _restoreIndexes) {
boost::filesystem::path metadataFile = (root.branch_path() / (oldCollName + ".metadata.json"));
if (!boost::filesystem::exists(metadataFile.string())) {
// This is fine because dumps from before 2.1 won't have a metadata file, just print a warning.
// System collections shouldn't have metadata so don't warn if that file is missing.
if (!startsWith(metadataFile.leaf().string(), "system.")) {
log() << metadataFile.string() << " not found. Skipping." << endl;
}
} else {
metadataObject = parseMetadataFile(metadataFile.string());
}
}
_curns = ns.c_str();
NamespaceString nss(_curns);
_curdb = nss.db;
_curcoll = nss.coll;
// If drop is not used, warn if the collection exists.
if (!_drop) {
scoped_ptr<DBClientCursor> cursor(conn().query(_curdb + ".system.namespaces",
Query(BSON("name" << ns))));
if (cursor->more()) {
// collection already exists show warning
warning() << "Restoring to " << ns << " without dropping. Restored data "
"will be inserted without raising errors; check your server log"
<< endl;
}
}
vector<BSONObj> indexes;
if (_restoreIndexes && metadataObject.hasField("indexes")) {
const vector<BSONElement> indexElements = metadataObject["indexes"].Array();
for (vector<BSONElement>::const_iterator it = indexElements.begin(); it != indexElements.end(); ++it) {
// Need to make sure the ns field gets updated to
// the proper _curdb + _curns value, if we're
// restoring to a different database.
const BSONObj indexObj = renameIndexNs(it->Obj());
indexes.push_back(indexObj);
}
}
const BSONObj options = _restoreOptions && metadataObject.hasField("options") ?
metadataObject["options"].Obj() : BSONObj();
if (_doBulkLoad) {
RemoteLoader loader(conn(), _curdb, _curcoll, indexes, options);
processFile( root );
loader.commit();
} else {
// No bulk load. Create collection and indexes manually.
if (!options.isEmpty()) {
createCollectionWithOptions(options);
}
// Build indexes last - it's a little faster.
processFile( root );
for (vector<BSONObj>::iterator it = indexes.begin(); it != indexes.end(); ++it) {
createIndex(*it);
}
}
if (_drop && root.leaf() == "system.users.bson") {
// Delete any users that used to exist but weren't in the dump file
for (set<string>::iterator it = _users.begin(); it != _users.end(); ++it) {
BSONObj userMatch = BSON("user" << *it);
conn().remove(ns, Query(userMatch));
}
_users.clear();
}
}
