void
manager::start_file(const string& file, client *cl, bool temp)
{
pthread_setspecific(client_key, cl);
scheduler_type sched_type(SCHED_SERIAL_UI);
try {
router rout(1, 0, NULL, is_mpi_sched(sched_type));
machine mac(file, rout, 1, sched_type);
cl->all = mac.get_all();
LOG_PROGRAM_RUNNING();
if(temp) {
delete_temp_file(file);
}
mac.start();
LOG_PROGRAM_STOPPED();
} catch(std::exception& e) {
cerr << e.what() << endl;
}
client_mtx.lock();
cl->all = NULL;
client_mtx.unlock();
}
void file_download_cache::release_cache(const std::string& url) {
// look for the file in the url_to_file map and delete it.
lock.lock();
if (url_to_file.count(url)) {
delete_temp_file(url_to_file[url].filename);
url_to_file.erase(url);
}
lock.unlock();
}
std::string unity_global::get_current_cache_file_location() {
auto the_file = get_temp_name();
boost::filesystem::path p(the_file);
if(!p.has_parent_path()) {
return std::string("");
}
auto the_location = p.parent_path();
delete_temp_file(the_file);
return the_location.string();
}
void cache_block::clear() {
if (data) {
logstream(LOG_DEBUG) << "Releasing cache ID " << cache_id << std::endl;
release_memory();
} else if (!filename.empty()) {
logstream(LOG_DEBUG) << "Releasing cache ID " << cache_id << std::endl;
// delete disk temp file
try {
logstream(LOG_DEBUG) << "Deleting cached file " << filename << std::endl;
delete_temp_file(filename);
} catch (...) {
logstream(LOG_WARNING) << "Failed to delete temporary file: "
<< filename << std::endl;
}
filename.clear();
}
}
static void
call_process_kill (void *ptr)
{
int *callproc_fd = ptr;
int i;
for (i = 0; i < CALLPROC_FDS; i++)
if (0 <= callproc_fd[i])
emacs_close (callproc_fd[i]);
if (synch_process_pid)
{
struct Lisp_Process proc;
proc.alive = 1;
proc.pid = synch_process_pid;
record_kill_process (&proc, synch_process_tempfile);
synch_process_pid = 0;
}
else if (STRINGP (synch_process_tempfile))
delete_temp_file (synch_process_tempfile);
}
std::string run_aws_command(const std::vector<std::string>& arglist,
const std::string& aws_access_key_id,
const std::string& aws_secret_access_key) {
{
std::lock_guard<graphlab::mutex> lock_guard(env_lock);
#ifndef _WIN32
setenv("AWS_ACCESS_KEY_ID", aws_access_key_id.c_str(), 1 /*overwrite*/);
setenv("AWS_SECRET_ACCESS_KEY", aws_secret_access_key.c_str(), 1 /*overwrite*/);
#else
_putenv_s("AWS_ACCESS_KEY_ID", aws_access_key_id.c_str());
_putenv_s("AWS_SECRET_ACCESS_KEY", aws_secret_access_key.c_str());
#endif
}
// Creates a temp file and redirect the child process's stderr
std::string child_err_file = get_temp_name();
std::stringstream command_builder;
std::vector<std::string> argv;
//TODO: Add a "launch_shell" function to process library
#ifndef _WIN32
std::string cmd = "/bin/sh";
argv.push_back("-c");
// We put cd here because aws command prints url relative to the working
// directory. Without cd, it will print out stuff like
// download s3://foo to ../../../../../../../../../var/tmp/graphlab/0001/foo
// with cd it will have less ".."'s. This is still not pretty.
command_builder << "cd && aws ";
#else
std::string cmd = "cmd.exe";
argv.push_back("/c");
command_builder << "aws ";
#endif
for (const auto& x: arglist)
command_builder << x << " ";
command_builder << "2>" << child_err_file;
argv.push_back(command_builder.str());
logstream(LOG_INFO) << "Running aws command: " << command_builder.str() << std::endl;
std::string ret;
process shell_proc;
shell_proc.popen(cmd, argv, STDOUT_FILENO);
auto progress_rc = wait_on_child_and_print_progress(shell_proc);
ret += get_child_error_or_empty(child_err_file);
delete_temp_file(child_err_file);
if(!progress_rc)
log_and_throw("Cancelled by user");
auto shell_rc = shell_proc.get_return_code();
if(shell_rc == 0) {
logstream(LOG_INFO) << "Succeeded with error message: " << ret << std::endl;
ret.clear();
}
return ret;
}
EXPORT void file_download_cache::clear() {
for(auto p: url_to_file) {
delete_temp_file(p.second.filename);
}
url_to_file.clear();
}
static void process_client(const char *node, const char *port, int fd)
{
char **temp_files;
char buf[BUFSIZ];
char *option;
int *port_array;
int *pid_array;
int pagesize;
int start_port;
int udp_port;
int options;
int size;
int cpus;
int cpu;
int pid;
int ofd;
int n, s, t, i;
/* Let the client know what we are */
write(fd, "tracecmd", 8);
/* read back the CPU count */
n = read_string(fd, buf, BUFSIZ);
if (n == BUFSIZ)
/** ERROR **/
return;
cpus = atoi(buf);
plog("cpus=%d\n", cpus);
if (cpus < 0)
return;
/* next read the page size */
n = read_string(fd, buf, BUFSIZ);
if (n == BUFSIZ)
/** ERROR **/
return;
pagesize = atoi(buf);
plog("pagesize=%d\n", pagesize);
if (pagesize <= 0)
return;
/* Now the number of options */
n = read_string(fd, buf, BUFSIZ);
if (n == BUFSIZ)
/** ERROR **/
return;
options = atoi(buf);
for (i = 0; i < options; i++) {
/* next is the size of the options */
n = read_string(fd, buf, BUFSIZ);
if (n == BUFSIZ)
/** ERROR **/
return;
size = atoi(buf);
/* prevent a client from killing us */
if (size > MAX_OPTION_SIZE)
return;
option = malloc_or_die(size);
do {
t = size;
s = 0;
s = read(fd, option+s, t);
if (s <= 0)
return;
t -= s;
s = size - t;
} while (t);
s = process_option(option);
free(option);
/* do we understand this option? */
if (!s)
return;
}
if (use_tcp)
plog("Using TCP for live connection\n");
/* Create the client file */
snprintf(buf, BUFSIZ, "%s.%s:%s.dat", output_file, node, port);
ofd = open(buf, O_RDWR | O_CREAT | O_TRUNC, 0644);
if (ofd < 0)
pdie("Can not create file %s", buf);
port_array = malloc_or_die(sizeof(int) * cpus);
pid_array = malloc_or_die(sizeof(int) * cpus);
memset(pid_array, 0, sizeof(int) * cpus);
start_port = START_PORT_SEARCH;
/* Now create a UDP port for each CPU */
for (cpu = 0; cpu < cpus; cpu++) {
udp_port = open_udp(node, port, &pid, cpu, pagesize, start_port);
if (udp_port < 0)
goto out_free;
port_array[cpu] = udp_port;
pid_array[cpu] = pid;
/* due to some bugging finding ports, force search after last port */
start_port = udp_port+1;
}
/* send the client a comma deliminated set of port numbers */
for (cpu = 0; cpu < cpus; cpu++) {
snprintf(buf, BUFSIZ, "%s%d",
cpu ? "," : "", port_array[cpu]);
write(fd, buf, strlen(buf));
}
/* end with null terminator */
write(fd, "\0", 1);
/* Now we are ready to start reading data from the client */
do {
n = read(fd, buf, BUFSIZ);
if (n < 0) {
if (errno == EINTR)
continue;
pdie("reading client");
}
t = n;
s = 0;
do {
s = write(ofd, buf+s, t);
if (s < 0) {
if (errno == EINTR)
break;
pdie("writing to file");
}
t -= s;
s = n - t;
} while (t);
} while (n > 0 && !done);
/* wait a little to let our readers finish reading */
sleep(1);
/* stop our readers */
for (cpu = 0; cpu < cpus; cpu++) {
if (pid_array[cpu] > 0)
kill(pid_array[cpu], SIGUSR1);
}
/* wait a little to have the readers clean up */
sleep(1);
/* Now put together the file */
temp_files = malloc_or_die(sizeof(*temp_files) * cpus);
for (cpu = 0; cpu < cpus; cpu++)
temp_files[cpu] = get_temp_file(node, port, cpu);
tracecmd_attach_cpu_data_fd(ofd, cpus, temp_files);
out_free:
for (cpu = 0; cpu < cpus; cpu++) {
if (pid_array[cpu] > 0) {
kill(pid_array[cpu], SIGKILL);
delete_temp_file(node, port, cpu);
pid_array[cpu] = 0;
}
}
}
