static gint32 load_image (const gchar *uri, GimpRunMode run_mode, GError **error) { gchar *tmpname = NULL; gint32 image_ID = -1; gboolean name_image = FALSE; tmpname = get_temp_name (uri, &name_image); if (uri_backend_load_image (uri, tmpname, run_mode, error)) { image_ID = gimp_file_load (run_mode, tmpname, tmpname); if (image_ID != -1) { if (name_image) gimp_image_set_filename (image_ID, uri); else gimp_image_set_filename (image_ID, ""); } else { g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED, "%s", gimp_get_pdb_error ()); } } g_unlink (tmpname); g_free (tmpname); return image_ID; }
static GimpPDBStatusType save_image (const gchar *uri, gint32 image_ID, gint32 drawable_ID, gint32 run_mode, GError **error) { GimpPDBStatusType status = GIMP_PDB_EXECUTION_ERROR; gchar *tmpname; tmpname = get_temp_name (uri, NULL); if (gimp_file_save (run_mode, image_ID, drawable_ID, tmpname, tmpname) && valid_file (tmpname)) { if (uri_backend_save_image (uri, tmpname, run_mode, error)) { status = GIMP_PDB_SUCCESS; } } else { g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED, "%s", gimp_get_pdb_error ()); } g_unlink (tmpname); g_free (tmpname); return status; }
graph_pylambda_master::graph_pylambda_master(size_t nworkers) { std::vector<std::string> worker_addresses; for (size_t i = 0; i < nworkers; ++i) { worker_addresses.push_back(std::string("ipc://") + get_temp_name()); } m_worker_pool.reset(new worker_pool<graph_lambda_evaluator_proxy>(nworkers, pylambda_worker_binary, worker_addresses)); }
static GimpPDBStatusType save_image (const gchar *uri, gint32 image_ID, gint32 drawable_ID, gint32 run_mode, GError **error) { GimpPDBStatusType status = GIMP_PDB_EXECUTION_ERROR; gchar *tmpname; gboolean mapped = FALSE; tmpname = uri_backend_map_image (uri, run_mode); if (tmpname) mapped = TRUE; else tmpname = get_temp_name (uri, NULL); if (gimp_file_save (run_mode, image_ID, drawable_ID, tmpname, tmpname)) { if (mapped) { status = GIMP_PDB_SUCCESS; } else if (valid_file (tmpname)) { if (uri_backend_save_image (uri, tmpname, run_mode, error)) { status = GIMP_PDB_SUCCESS; } } else { g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED, _("Failed to save to temporary file '%s'"), gimp_filename_to_utf8 (tmpname)); } } else { g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED, "%s", gimp_get_pdb_error ()); } if (! mapped) g_unlink (tmpname); g_free (tmpname); return status; }
std::string unity_global::get_current_cache_file_location() { auto the_file = get_temp_name(); boost::filesystem::path p(the_file); if(!p.has_parent_path()) { return std::string(""); } auto the_location = p.parent_path(); delete_temp_file(the_file); return the_location.string(); }
static gint32 load_image (const gchar *uri, GimpRunMode run_mode, GError **error) { gint32 image_ID = -1; gboolean name_image = FALSE; gchar *tmpname; gboolean mapped = FALSE; tmpname = uri_backend_map_image (uri, run_mode); if (tmpname) { mapped = TRUE; } else { tmpname = get_temp_name (uri, &name_image); if (! uri_backend_load_image (uri, tmpname, run_mode, error)) return -1; } image_ID = gimp_file_load (run_mode, tmpname, tmpname); if (image_ID != -1) { if (mapped || name_image) gimp_image_set_filename (image_ID, uri); else gimp_image_set_filename (image_ID, ""); } else { g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_FAILED, "%s", gimp_get_pdb_error ()); } if (! mapped) g_unlink (tmpname); g_free (tmpname); return image_ID; }
graph_pylambda_master::graph_pylambda_master(size_t nworkers) { std::vector<std::string> worker_addresses; for (size_t i = 0; i < nworkers; ++i) { worker_addresses.push_back(std::string("ipc://") + get_temp_name()); } m_worker_pool.reset( new worker_pool<graph_lambda_evaluator_proxy>( nworkers, lambda_master::get_lambda_worker_binary(), worker_addresses)); if (nworkers < thread::cpu_count()) { logprogress_stream << "Using default " << nworkers << " lambda workers.\n"; logprogress_stream << "To maximize the degree of parallelism, add the following code to the beginning of the program:\n"; logprogress_stream << "\"graphlab.set_runtime_config(\'GRAPHLAB_DEFAULT_NUM_GRAPH_LAMBDA_WORKERS\', " << thread::cpu_count() << ")\"\n"; logprogress_stream << "Note that increasing the degree of parallelism also increases the memory footprint." << std::endl; } }
std::string unity_global::load_toolkit(std::string soname, std::string module_subpath) { // rewrite "local" protocol std::string protocol = fileio::get_protocol(soname); if (protocol == "local") { soname = fileio::remove_protocol(soname); } so_registration_list regentry; regentry.original_soname = soname; logstream(LOG_INFO) << "Attempt loading of " << sanitize_url(soname) << std::endl; // see if the file exists and whether we need to donwnload it if (fileio::try_to_open_file(soname) == false) { return "Unable to open file " + sanitize_url(soname); } if (protocol != "") { // there is a protocol associated. We need to copy this file to local // issue a copy to copy it to the local temp directory std::string tempname = get_temp_name(); fileio::copy(soname, tempname); soname = tempname; } if (!file_contains_substring(soname, "get_toolkit_function_registration") && !file_contains_substring(soname, "get_toolkit_class_registration")) { return soname + " is not a valid extension"; } // get the base name of the shared library (without the .so) std::string modulename = fileio::get_filename(regentry.original_soname); std::vector<std::string> split_names; boost::algorithm::split(split_names, modulename, boost::is_any_of(".")); if (split_names.size() == 0) return "Invalid filename"; if (module_subpath.empty()) { regentry.modulename = split_names[0]; } else if (module_subpath == "..") { regentry.modulename = ""; } else { regentry.modulename = module_subpath + "." + split_names[0]; } // goody. now for the dl loading #ifndef _WIN32 void* dl = dlopen(soname.c_str(), RTLD_NOW | RTLD_LOCAL); #else void *dl = (void *)LoadLibrary(soname.c_str()); #endif logstream(LOG_INFO) << "Library load of " << sanitize_url(soname) << std::endl; regentry.effective_soname = soname; regentry.dl = dl; // check for failure if (dl == NULL) { #ifndef _WIN32 char* err = dlerror(); // I think we need to copy this out early std::string ret = err; logstream(LOG_ERROR) << "Unable to load " << sanitize_url(soname) << ": " << ret << std::endl; if (err) return ret; else return "dlopen failed due to an unknown error"; #else std::string ret = get_last_err_str(GetLastError()); logstream(LOG_ERROR) << "Unable to load " << sanitize_url(soname) << ": " << ret << std::endl; if (!ret.empty()) return ret; else return "LoadLibrary failed due to an unknown error"; #endif } /**************************************************************************/ /* */ /* Function Registration */ /* */ /**************************************************************************/ // get the registration symbols std::vector<std::string> toolkit_function_reg_names {"get_toolkit_function_registration", "_Z33get_toolkit_function_registrationv", "__Z33get_toolkit_function_registrationv"}; get_toolkit_function_registration_type get_toolkit_function_registration = nullptr; for (auto reg_name : toolkit_function_reg_names) { get_toolkit_function_registration = reinterpret_cast<get_toolkit_function_registration_type> ( #ifndef _WIN32 dlsym(dl, reg_name.c_str()) #else (void *)GetProcAddress((HMODULE)dl, reg_name.c_str()) #endif ); if (get_toolkit_function_registration != nullptr) break; } // register functions if (get_toolkit_function_registration) { auto functions = (*get_toolkit_function_registration)(); for (auto& fn: functions) { if (!regentry.modulename.empty()) { fn.name = regentry.modulename + "." + fn.name; } fn.description["file"] = regentry.original_soname; logstream(LOG_INFO) << "Adding function: " << fn.name << std::endl; regentry.functions.push_back(fn.name); } toolkit_functions->register_toolkit_function(functions); } /**************************************************************************/ /* */ /* Class Registration */ /* */ /**************************************************************************/ std::vector<std::string> toolkit_class_reg_names {"get_toolkit_class_registration", "_Z30get_toolkit_class_registrationv", "__Z30get_toolkit_class_registrationv"}; get_toolkit_class_registration_type get_toolkit_class_registration = nullptr; for (auto reg_name : toolkit_class_reg_names) { get_toolkit_class_registration = reinterpret_cast<get_toolkit_class_registration_type> ( #ifndef _WIN32 dlsym(dl, reg_name.c_str()) #else (void *)GetProcAddress((HMODULE)dl, reg_name.c_str()) #endif ); if (get_toolkit_class_registration != nullptr) break; } // register classes if (get_toolkit_class_registration) { auto class_reg = (*get_toolkit_class_registration)(); for (auto& cl: class_reg) { if (!regentry.modulename.empty()) { cl.name = regentry.modulename + "." + cl.name; } cl.description["file"] = regentry.original_soname; logstream(LOG_INFO) << "Adding class : " << cl.name << std::endl; regentry.functions.push_back(cl.name); } classes->register_toolkit_class(class_reg); } if (regentry.functions.empty() && regentry.classes.empty()) { // nothing has been registered! unload the dl #ifndef _WIN32 dlclose(dl); #else FreeLibrary((HMODULE)dl); #endif return "No functions or classes registered by " + sanitize_url(soname); } // note that it is possible to load a toolkit multiple times. // It is not safe to unload previously loaded toolkits since I may have // a reference to it (for instance a class). We just keep loading over // and hope for the best. // store and remember the dlhandle and what was registered; dynamic_loaded_toolkits[regentry.original_soname] = regentry; return std::string(); }
std::string run_aws_command(const std::vector<std::string>& arglist, const std::string& aws_access_key_id, const std::string& aws_secret_access_key) { { std::lock_guard<graphlab::mutex> lock_guard(env_lock); #ifndef _WIN32 setenv("AWS_ACCESS_KEY_ID", aws_access_key_id.c_str(), 1 /*overwrite*/); setenv("AWS_SECRET_ACCESS_KEY", aws_secret_access_key.c_str(), 1 /*overwrite*/); #else _putenv_s("AWS_ACCESS_KEY_ID", aws_access_key_id.c_str()); _putenv_s("AWS_SECRET_ACCESS_KEY", aws_secret_access_key.c_str()); #endif } // Creates a temp file and redirect the child process's stderr std::string child_err_file = get_temp_name(); std::stringstream command_builder; std::vector<std::string> argv; //TODO: Add a "launch_shell" function to process library #ifndef _WIN32 std::string cmd = "/bin/sh"; argv.push_back("-c"); // We put cd here because aws command prints url relative to the working // directory. Without cd, it will print out stuff like // download s3://foo to ../../../../../../../../../var/tmp/graphlab/0001/foo // with cd it will have less ".."'s. This is still not pretty. command_builder << "cd && aws "; #else std::string cmd = "cmd.exe"; argv.push_back("/c"); command_builder << "aws "; #endif for (const auto& x: arglist) command_builder << x << " "; command_builder << "2>" << child_err_file; argv.push_back(command_builder.str()); logstream(LOG_INFO) << "Running aws command: " << command_builder.str() << std::endl; std::string ret; process shell_proc; shell_proc.popen(cmd, argv, STDOUT_FILENO); auto progress_rc = wait_on_child_and_print_progress(shell_proc); ret += get_child_error_or_empty(child_err_file); delete_temp_file(child_err_file); if(!progress_rc) log_and_throw("Cancelled by user"); auto shell_rc = shell_proc.get_return_code(); if(shell_rc == 0) { logstream(LOG_INFO) << "Succeeded with error message: " << ret << std::endl; ret.clear(); } return ret; }
std::string get_temp_name_prefer_hdfs(const std::string& prefix) { bool prefer_hdfs = true; return get_temp_name(prefix, prefer_hdfs); }
std::string file_download_cache::get_file(const std::string& url) { // first check if the file has been downloaded. // if it has, return the downloaded location lock.lock(); if (url_to_file.count(url)) { bool cache_dirty = false; if (boost::starts_with(url, "s3://")) { std::string last_modified = ""; try { last_modified = webstor::get_s3_file_last_modified(url); } catch (...) { lock.unlock(); throw; } if (last_modified != url_to_file[url].last_modified) { cache_dirty = true; } } if (!cache_dirty) { std::string ret = url_to_file[url].filename; lock.unlock(); return ret; } } lock.unlock(); // ok. we need to download the file if (boost::starts_with(url, "s3://")) { // if it is s3. std::string localfile = get_temp_name(); std::string message = webstor::download_from_s3(url, localfile, "").get(); size_t i = 0; // if message contains a permanentredirect error code, we need to try other endpoints. while (boost::algorithm::icontains(message, "PermanentRedirect") && i < webstor::S3_END_POINTS.size()) { message = webstor::download_from_s3(url, localfile, "", webstor::S3_END_POINTS[i]).get(); ++i; } if (!message.empty()) { // OK, we failed. Let's clean up anything that was downloaded // since the real file lives in S3. if(std::remove(localfile.c_str()) != 0) { logstream(LOG_WARNING) << "Could not delete failed cached file: " << localfile << std::endl; } log_and_throw_io_failure("Fail to download from " + webstor::sanitize_s3_url(url) + ". " + webstor::get_s3_error_code(message)); } lock.lock(); url_to_file[url].filename = localfile; try { url_to_file[url].last_modified = webstor::get_s3_file_last_modified(url); } catch (...) { lock.unlock(); throw; } lock.unlock(); return localfile; } else { // Ok, it is either local regular file, file:///, or remote urls http://. // For remote urls, download_url download it into to local file. // For local urls, download_url return as is. std::string localfile; int status; bool is_temp; std::tie(status, is_temp, localfile) = download_url(url); if (status) { log_and_throw_io_failure("Fail to download from " + url + ". " + get_curl_error_string(status)); } if (is_temp) { // if it is a remote file, we check the download status code lock.lock(); url_to_file[url].filename = localfile; url_to_file[url].last_modified = ""; lock.unlock(); return localfile; } else { // purely a local file. just return it return localfile; } } }