static gint32
load_image (const gchar *uri,
GimpRunMode run_mode,
GError **error)
{
gchar *tmpname = NULL;
gint32 image_ID = -1;
gboolean name_image = FALSE;
tmpname = get_temp_name (uri, &name_image);
if (uri_backend_load_image (uri, tmpname, run_mode, error))
{
image_ID = gimp_file_load (run_mode, tmpname, tmpname);
if (image_ID != -1)
{
if (name_image)
gimp_image_set_filename (image_ID, uri);
else
gimp_image_set_filename (image_ID, "");
}
else
{
g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
"%s", gimp_get_pdb_error ());
}
}
g_unlink (tmpname);
g_free (tmpname);
return image_ID;
}
static GimpPDBStatusType
save_image (const gchar *uri,
gint32 image_ID,
gint32 drawable_ID,
gint32 run_mode,
GError **error)
{
GimpPDBStatusType status = GIMP_PDB_EXECUTION_ERROR;
gchar *tmpname;
tmpname = get_temp_name (uri, NULL);
if (gimp_file_save (run_mode,
image_ID,
drawable_ID,
tmpname,
tmpname) && valid_file (tmpname))
{
if (uri_backend_save_image (uri, tmpname, run_mode, error))
{
status = GIMP_PDB_SUCCESS;
}
}
else
{
g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
"%s", gimp_get_pdb_error ());
}
g_unlink (tmpname);
g_free (tmpname);
return status;
}
graph_pylambda_master::graph_pylambda_master(size_t nworkers) {
std::vector<std::string> worker_addresses;
for (size_t i = 0; i < nworkers; ++i) {
worker_addresses.push_back(std::string("ipc://") + get_temp_name());
}
m_worker_pool.reset(new worker_pool<graph_lambda_evaluator_proxy>(nworkers,
pylambda_worker_binary,
worker_addresses));
}
static GimpPDBStatusType
save_image (const gchar *uri,
gint32 image_ID,
gint32 drawable_ID,
gint32 run_mode,
GError **error)
{
GimpPDBStatusType status = GIMP_PDB_EXECUTION_ERROR;
gchar *tmpname;
gboolean mapped = FALSE;
tmpname = uri_backend_map_image (uri, run_mode);
if (tmpname)
mapped = TRUE;
else
tmpname = get_temp_name (uri, NULL);
if (gimp_file_save (run_mode,
image_ID,
drawable_ID,
tmpname,
tmpname))
{
if (mapped)
{
status = GIMP_PDB_SUCCESS;
}
else if (valid_file (tmpname))
{
if (uri_backend_save_image (uri, tmpname, run_mode, error))
{
status = GIMP_PDB_SUCCESS;
}
}
else
{
g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
_("Failed to save to temporary file '%s'"),
gimp_filename_to_utf8 (tmpname));
}
}
else
{
g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
"%s", gimp_get_pdb_error ());
}
if (! mapped)
g_unlink (tmpname);
g_free (tmpname);
return status;
}
std::string unity_global::get_current_cache_file_location() {
auto the_file = get_temp_name();
boost::filesystem::path p(the_file);
if(!p.has_parent_path()) {
return std::string("");
}
auto the_location = p.parent_path();
delete_temp_file(the_file);
return the_location.string();
}
static gint32
load_image (const gchar *uri,
GimpRunMode run_mode,
GError **error)
{
gint32 image_ID = -1;
gboolean name_image = FALSE;
gchar *tmpname;
gboolean mapped = FALSE;
tmpname = uri_backend_map_image (uri, run_mode);
if (tmpname)
{
mapped = TRUE;
}
else
{
tmpname = get_temp_name (uri, &name_image);
if (! uri_backend_load_image (uri, tmpname, run_mode, error))
return -1;
}
image_ID = gimp_file_load (run_mode, tmpname, tmpname);
if (image_ID != -1)
{
if (mapped || name_image)
gimp_image_set_filename (image_ID, uri);
else
gimp_image_set_filename (image_ID, "");
}
else
{
g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED,
"%s", gimp_get_pdb_error ());
}
if (! mapped)
g_unlink (tmpname);
g_free (tmpname);
return image_ID;
}
graph_pylambda_master::graph_pylambda_master(size_t nworkers) {
std::vector<std::string> worker_addresses;
for (size_t i = 0; i < nworkers; ++i) {
worker_addresses.push_back(std::string("ipc://") + get_temp_name());
}
m_worker_pool.reset(
new worker_pool<graph_lambda_evaluator_proxy>(
nworkers,
lambda_master::get_lambda_worker_binary(),
worker_addresses));
if (nworkers < thread::cpu_count()) {
logprogress_stream << "Using default " << nworkers << " lambda workers.\n";
logprogress_stream << "To maximize the degree of parallelism, add the following code to the beginning of the program:\n";
logprogress_stream << "\"graphlab.set_runtime_config(\'GRAPHLAB_DEFAULT_NUM_GRAPH_LAMBDA_WORKERS\', " << thread::cpu_count() << ")\"\n";
logprogress_stream << "Note that increasing the degree of parallelism also increases the memory footprint." << std::endl;
}
}
std::string unity_global::load_toolkit(std::string soname,
std::string module_subpath) {
// rewrite "local" protocol
std::string protocol = fileio::get_protocol(soname);
if (protocol == "local") {
soname = fileio::remove_protocol(soname);
}
so_registration_list regentry;
regentry.original_soname = soname;
logstream(LOG_INFO) << "Attempt loading of " << sanitize_url(soname) << std::endl;
// see if the file exists and whether we need to donwnload it
if (fileio::try_to_open_file(soname) == false) {
return "Unable to open file " + sanitize_url(soname);
}
if (protocol != "") {
// there is a protocol associated. We need to copy this file to local
// issue a copy to copy it to the local temp directory
std::string tempname = get_temp_name();
fileio::copy(soname, tempname);
soname = tempname;
}
if (!file_contains_substring(soname, "get_toolkit_function_registration") &&
!file_contains_substring(soname, "get_toolkit_class_registration")) {
return soname + " is not a valid extension";
}
// get the base name of the shared library (without the .so)
std::string modulename = fileio::get_filename(regentry.original_soname);
std::vector<std::string> split_names;
boost::algorithm::split(split_names, modulename, boost::is_any_of("."));
if (split_names.size() == 0) return "Invalid filename";
if (module_subpath.empty()) {
regentry.modulename = split_names[0];
} else if (module_subpath == "..") {
regentry.modulename = "";
} else {
regentry.modulename = module_subpath + "." + split_names[0];
}
// goody. now for the dl loading
#ifndef _WIN32
void* dl = dlopen(soname.c_str(), RTLD_NOW | RTLD_LOCAL);
#else
void *dl = (void *)LoadLibrary(soname.c_str());
#endif
logstream(LOG_INFO) << "Library load of " << sanitize_url(soname) << std::endl;
regentry.effective_soname = soname;
regentry.dl = dl;
// check for failure
if (dl == NULL) {
#ifndef _WIN32
char* err = dlerror();
// I think we need to copy this out early
std::string ret = err;
logstream(LOG_ERROR) << "Unable to load " << sanitize_url(soname) << ": " << ret << std::endl;
if (err) return ret;
else return "dlopen failed due to an unknown error";
#else
std::string ret = get_last_err_str(GetLastError());
logstream(LOG_ERROR) << "Unable to load " << sanitize_url(soname) << ": " << ret << std::endl;
if (!ret.empty()) return ret;
else return "LoadLibrary failed due to an unknown error";
#endif
}
/**************************************************************************/
/* */
/* Function Registration */
/* */
/**************************************************************************/
// get the registration symbols
std::vector<std::string> toolkit_function_reg_names
{"get_toolkit_function_registration",
"_Z33get_toolkit_function_registrationv",
"__Z33get_toolkit_function_registrationv"};
get_toolkit_function_registration_type get_toolkit_function_registration = nullptr;
for (auto reg_name : toolkit_function_reg_names) {
get_toolkit_function_registration =
reinterpret_cast<get_toolkit_function_registration_type>
(
#ifndef _WIN32
dlsym(dl, reg_name.c_str())
#else
(void *)GetProcAddress((HMODULE)dl, reg_name.c_str())
#endif
);
if (get_toolkit_function_registration != nullptr) break;
}
// register functions
if (get_toolkit_function_registration) {
auto functions = (*get_toolkit_function_registration)();
for (auto& fn: functions) {
if (!regentry.modulename.empty()) {
fn.name = regentry.modulename + "." + fn.name;
}
fn.description["file"] = regentry.original_soname;
logstream(LOG_INFO) << "Adding function: " << fn.name << std::endl;
regentry.functions.push_back(fn.name);
}
toolkit_functions->register_toolkit_function(functions);
}
/**************************************************************************/
/* */
/* Class Registration */
/* */
/**************************************************************************/
std::vector<std::string> toolkit_class_reg_names
{"get_toolkit_class_registration",
"_Z30get_toolkit_class_registrationv",
"__Z30get_toolkit_class_registrationv"};
get_toolkit_class_registration_type get_toolkit_class_registration = nullptr;
for (auto reg_name : toolkit_class_reg_names) {
get_toolkit_class_registration =
reinterpret_cast<get_toolkit_class_registration_type>
(
#ifndef _WIN32
dlsym(dl, reg_name.c_str())
#else
(void *)GetProcAddress((HMODULE)dl, reg_name.c_str())
#endif
);
if (get_toolkit_class_registration != nullptr) break;
}
// register classes
if (get_toolkit_class_registration) {
auto class_reg = (*get_toolkit_class_registration)();
for (auto& cl: class_reg) {
if (!regentry.modulename.empty()) {
cl.name = regentry.modulename + "." + cl.name;
}
cl.description["file"] = regentry.original_soname;
logstream(LOG_INFO) << "Adding class : " << cl.name << std::endl;
regentry.functions.push_back(cl.name);
}
classes->register_toolkit_class(class_reg);
}
if (regentry.functions.empty() && regentry.classes.empty()) {
// nothing has been registered! unload the dl
#ifndef _WIN32
dlclose(dl);
#else
FreeLibrary((HMODULE)dl);
#endif
return "No functions or classes registered by " + sanitize_url(soname);
}
// note that it is possible to load a toolkit multiple times.
// It is not safe to unload previously loaded toolkits since I may have
// a reference to it (for instance a class). We just keep loading over
// and hope for the best.
// store and remember the dlhandle and what was registered;
dynamic_loaded_toolkits[regentry.original_soname] = regentry;
return std::string();
}
std::string run_aws_command(const std::vector<std::string>& arglist,
const std::string& aws_access_key_id,
const std::string& aws_secret_access_key) {
{
std::lock_guard<graphlab::mutex> lock_guard(env_lock);
#ifndef _WIN32
setenv("AWS_ACCESS_KEY_ID", aws_access_key_id.c_str(), 1 /*overwrite*/);
setenv("AWS_SECRET_ACCESS_KEY", aws_secret_access_key.c_str(), 1 /*overwrite*/);
#else
_putenv_s("AWS_ACCESS_KEY_ID", aws_access_key_id.c_str());
_putenv_s("AWS_SECRET_ACCESS_KEY", aws_secret_access_key.c_str());
#endif
}
// Creates a temp file and redirect the child process's stderr
std::string child_err_file = get_temp_name();
std::stringstream command_builder;
std::vector<std::string> argv;
//TODO: Add a "launch_shell" function to process library
#ifndef _WIN32
std::string cmd = "/bin/sh";
argv.push_back("-c");
// We put cd here because aws command prints url relative to the working
// directory. Without cd, it will print out stuff like
// download s3://foo to ../../../../../../../../../var/tmp/graphlab/0001/foo
// with cd it will have less ".."'s. This is still not pretty.
command_builder << "cd && aws ";
#else
std::string cmd = "cmd.exe";
argv.push_back("/c");
command_builder << "aws ";
#endif
for (const auto& x: arglist)
command_builder << x << " ";
command_builder << "2>" << child_err_file;
argv.push_back(command_builder.str());
logstream(LOG_INFO) << "Running aws command: " << command_builder.str() << std::endl;
std::string ret;
process shell_proc;
shell_proc.popen(cmd, argv, STDOUT_FILENO);
auto progress_rc = wait_on_child_and_print_progress(shell_proc);
ret += get_child_error_or_empty(child_err_file);
delete_temp_file(child_err_file);
if(!progress_rc)
log_and_throw("Cancelled by user");
auto shell_rc = shell_proc.get_return_code();
if(shell_rc == 0) {
logstream(LOG_INFO) << "Succeeded with error message: " << ret << std::endl;
ret.clear();
}
return ret;
}
std::string get_temp_name_prefer_hdfs(const std::string& prefix) {
bool prefer_hdfs = true;
return get_temp_name(prefix, prefer_hdfs);
}
std::string file_download_cache::get_file(const std::string& url) {
// first check if the file has been downloaded.
// if it has, return the downloaded location
lock.lock();
if (url_to_file.count(url)) {
bool cache_dirty = false;
if (boost::starts_with(url, "s3://")) {
std::string last_modified = "";
try {
last_modified = webstor::get_s3_file_last_modified(url);
} catch (...) {
lock.unlock();
throw;
}
if (last_modified != url_to_file[url].last_modified) {
cache_dirty = true;
}
}
if (!cache_dirty) {
std::string ret = url_to_file[url].filename;
lock.unlock();
return ret;
}
}
lock.unlock();
// ok. we need to download the file
if (boost::starts_with(url, "s3://")) {
// if it is s3.
std::string localfile = get_temp_name();
std::string message = webstor::download_from_s3(url, localfile, "").get();
size_t i = 0;
// if message contains a permanentredirect error code, we need to try other endpoints.
while (boost::algorithm::icontains(message, "PermanentRedirect") && i < webstor::S3_END_POINTS.size()) {
message = webstor::download_from_s3(url, localfile, "", webstor::S3_END_POINTS[i]).get();
++i;
}
if (!message.empty()) {
// OK, we failed. Let's clean up anything that was downloaded
// since the real file lives in S3.
if(std::remove(localfile.c_str()) != 0) {
logstream(LOG_WARNING) << "Could not delete failed cached file: " << localfile << std::endl;
}
log_and_throw_io_failure("Fail to download from " + webstor::sanitize_s3_url(url) + ". "
+ webstor::get_s3_error_code(message));
}
lock.lock();
url_to_file[url].filename = localfile;
try {
url_to_file[url].last_modified = webstor::get_s3_file_last_modified(url);
} catch (...) {
lock.unlock();
throw;
}
lock.unlock();
return localfile;
} else {
// Ok, it is either local regular file, file:///, or remote urls http://.
// For remote urls, download_url download it into to local file.
// For local urls, download_url return as is.
std::string localfile;
int status; bool is_temp;
std::tie(status, is_temp, localfile) = download_url(url);
if (status) {
log_and_throw_io_failure("Fail to download from " + url +
". " + get_curl_error_string(status));
}
if (is_temp) {
// if it is a remote file, we check the download status code
lock.lock();
url_to_file[url].filename = localfile;
url_to_file[url].last_modified = "";
lock.unlock();
return localfile;
} else {
// purely a local file. just return it
return localfile;
}
}
}
