void do_register(acetables *g_ape)
{
register_cmd("CONNECT", cmd_connect, NEED_NOTHING, g_ape);
register_cmd("SCRIPT", cmd_script, NEED_NOTHING, g_ape);
register_cmd("CHECK", cmd_check, NEED_SESSID, g_ape);
register_cmd("SEND", cmd_send, NEED_SESSID, g_ape);
register_cmd("QUIT", cmd_quit, NEED_SESSID, g_ape);
register_cmd("JOIN", cmd_join, NEED_SESSID, g_ape);
register_cmd("LEFT", cmd_left, NEED_SESSID, g_ape);
register_cmd("SESSION", cmd_session, NEED_SESSID, g_ape);
}
void do_register(acetables *g_ape) // register_raw("CMD", Nparam (without IP and time, with sessid), callback_func, NEEDSOMETHING?, g_ape);
{
register_cmd("CONNECT", 1, cmd_connect, NEED_NOTHING, g_ape);
register_cmd("PCONNECT", 1, cmd_pconnect, NEED_NOTHING, g_ape);
register_cmd("SCRIPT", -1, cmd_script, NEED_NOTHING, g_ape);
register_cmd("CHECK", 1, cmd_check, NEED_SESSID, g_ape);
register_cmd("SEND", 3, cmd_send, NEED_SESSID, g_ape);
register_cmd("QUIT", 1, cmd_quit, NEED_SESSID, g_ape);
register_cmd("SETLEVEL", 4, cmd_setlevel, NEED_SESSID, g_ape); // Module
register_cmd("SETTOPIC", 3, cmd_settopic, NEED_SESSID, g_ape); // Module
register_cmd("JOIN", 2, cmd_join, NEED_SESSID, g_ape);
register_cmd("LEFT", 2, cmd_left, NEED_SESSID, g_ape);
register_cmd("KICK", 3, cmd_kick, NEED_SESSID, g_ape); // Module
register_cmd("BAN", 5, cmd_ban, NEED_SESSID, g_ape); // Module
register_cmd("SESSION", -3, cmd_session, NEED_SESSID, g_ape);
register_cmd("KONG", 2, cmd_pong, NEED_SESSID, g_ape);
register_cmd("PROXY_CONNECT", 3, cmd_proxy_connect, NEED_SESSID, g_ape);
register_cmd("PROXY_WRITE", 3, cmd_proxy_write, NEED_SESSID, g_ape);
}
int main(int argc, char **argv)
{
int RC;
pthread_attr_t attr;
default_pthead_attr(&attr);
pthread_mutex_init(&keep_alive_mutex, NULL);
/* Lock the keep-alive mutex */
pthread_mutex_lock(&keep_alive_mutex);
/* Allocate a new parallel_wrapper structure */
parallel_wrapper *par_wrapper = (parallel_wrapper *)calloc(1, sizeof(struct parallel_wrapper));
if (par_wrapper == (parallel_wrapper *)NULL)
{
print(PRNT_ERR, "Unable to allocate space for parallel wrapper\n");
return 1;
}
/* Install command signal handlers */
signal(SIGINT, handle_exit_signal);
signal(SIGTERM, handle_exit_signal);
signal(SIGHUP, handle_exit_signal);
/* Create structures for this machine */
par_wrapper -> this_machine = calloc(1, sizeof(struct machine));
if (par_wrapper -> this_machine == (machine *)NULL)
{
print(PRNT_ERR, "Unable to allocate space for this machine\n");
return 1;
}
/* Fill in the default values for port ranges */
par_wrapper -> low_port = LOW_PORT;
par_wrapper -> high_port = HIGH_PORT;
/* Fill in other default values */
par_wrapper -> this_machine -> rank = -1;
par_wrapper -> num_procs = -1;
par_wrapper -> command_socket = -1;
par_wrapper -> pgid = -1;
par_wrapper -> ka_interval = KA_INTERVAL;
par_wrapper -> timeout = TIMEOUT;
/* Default mutex state */
pthread_mutex_init(&par_wrapper -> mutex, NULL);
/* Allocate a list of symlinks */
par_wrapper -> symlinks = sll_get_list();
/* Get the initial working directory */
par_wrapper -> this_machine -> iwd = getcwd(NULL, 0); /* Allocates space */
/* Determine our name */
uid_t uid = getuid();
struct passwd *user_stats = getpwuid(uid);
if (user_stats == (struct passwd *)NULL)
{
print(PRNT_WARN, "Unable to determine the name of the current user - assuming 'nobody'\n");
par_wrapper -> this_machine -> user = strdup("nobody");
}
else
{
par_wrapper -> this_machine -> user = strdup(user_stats -> pw_name);
}
/* Parse environment variables and command line arguments */
parse_environment_vars(par_wrapper);
parse_args(argc, argv, par_wrapper);
/* Set the environment variables */
set_environment_vars(par_wrapper);
/* Check that required values are filled in */
if (par_wrapper -> this_machine -> rank < 0)
{
print(PRNT_ERR, "Invalid rank (%d). Environment variable or command option not set.\n",
par_wrapper -> this_machine -> rank);
return 2;
}
if (par_wrapper -> num_procs < 0)
{
print(PRNT_ERR, "Invalid number of processors (%d). Environment variable or command option not set.\n",
par_wrapper -> num_procs);
return 2;
}
if (par_wrapper -> timeout <= (2*par_wrapper -> ka_interval))
{
print(PRNT_WARN, "Keep-alive interval and timeout too close. Using default values.\n");
par_wrapper -> timeout = TIMEOUT;
par_wrapper -> ka_interval = KA_INTERVAL;
}
/* Get the IP address for this machine */
par_wrapper -> this_machine -> ip_addr = get_ip_addr();
debug(PRNT_INFO, "IP Addr: %s\n", par_wrapper -> this_machine -> ip_addr);
if (par_wrapper -> this_machine -> ip_addr == (char *)NULL)
{
print(PRNT_ERR, "Unable to get the IP address for this machine\n");
return 2;
}
/* Get a command port for this machine */
RC = get_bound_dgram_socket_by_range(par_wrapper -> low_port,
par_wrapper -> high_port, &par_wrapper -> this_machine -> port,
&par_wrapper -> command_socket);
if (RC != 0)
{
print(PRNT_ERR, "Unable to bind to command socket\n");
return 2;
}
else
{
debug(PRNT_INFO, "Bound to command port: %d\n", par_wrapper -> this_machine -> port);
}
/**
* If this is rank 0, point rank 0 to this_machine, otherwise allocate
* a new structure for the master
*/
if (par_wrapper -> this_machine -> rank == MASTER)
{
par_wrapper -> master = par_wrapper -> this_machine;
par_wrapper -> machines = (machine **) calloc(par_wrapper -> num_procs, sizeof(machine *));
if (par_wrapper -> machines == (machine **)NULL)
{
print(PRNT_ERR, "Unable to allocate space for machines array\n");
return 3;
}
par_wrapper -> machines[0] = par_wrapper -> master;
}
else
{
par_wrapper -> master = (machine *)calloc(1, sizeof(struct machine));
if (par_wrapper -> master == (machine *)NULL)
{
print(PRNT_ERR, "Unable to allocate space for master\n");
return 2;
}
par_wrapper -> master -> rank = MASTER;
}
/* Create the scratch directory */
create_scratch(par_wrapper);
/* Gather the necessary chirp information */
RC = chirp_info(par_wrapper);
if (RC != 0)
{
print(PRNT_ERR, "Failure sending/recieving chirp information\n");
return 2;
}
/* Create the listener */
pthread_create(&par_wrapper -> listener, &attr, &udp_server, (void *)par_wrapper);
/* If I am the MASTER, wait for all ranks to register */
if (par_wrapper -> this_machine -> rank == MASTER)
{
int i, j;
j = 0; /* Timeout = 0 */
while ( 1 )
{
int finished = 1;
for (i = 0; i < par_wrapper -> num_procs; i++)
{
if (par_wrapper -> machines[i] == NULL)
{
finished = 0;
}
if ((j % par_wrapper -> ka_interval) == 0 && par_wrapper -> machines[i] == NULL)
{
debug(PRNT_INFO, "Waiting for registration from rank %d\n", i);
}
}
j++; /* Waiting for timeout */
if (j >= par_wrapper -> timeout)
{
finished = 1;
for (i = 0; i < par_wrapper -> num_procs; i++)
{
if (par_wrapper -> machines[i] == NULL)
{
print(PRNT_WARN, "Rank %d not registered - giving up on it\n", i);
}
}
}
if (finished == 1)
{
break;
}
sleep(1);
}
debug(PRNT_INFO, "Finished machine registration.\n", par_wrapper -> num_procs);
/* Create the machines file */
RC = create_machine_file(par_wrapper);
if (RC != 0)
{
print(PRNT_ERR, "Unable to create the machines files");
cleanup(par_wrapper, 5);
}
RC = create_ssh_config(par_wrapper);
if (RC != 0)
{
print(PRNT_ERR, "Unable to create the machines files");
cleanup(par_wrapper, 5);
}
}
else /* Register with the master */
{
struct timeval old_time = par_wrapper -> master -> last_alive;
while ( 1 )
{
RC = register_cmd(par_wrapper -> command_socket, par_wrapper -> this_machine -> rank,
par_wrapper -> this_machine -> cpus,
par_wrapper -> this_machine -> iwd, par_wrapper -> this_machine -> user, par_wrapper -> master -> ip_addr,
par_wrapper -> master -> port);
sleep(1);
if (RC == 0 &&
((old_time.tv_sec != par_wrapper -> master -> last_alive.tv_sec) ||
(old_time.tv_usec != par_wrapper -> master -> last_alive.tv_usec)))
{
break;
}
debug(PRNT_INFO, "Waiting for ACK from mater\n");
/* Retrieve chirp information (in case it changed since the last time) */
chirp_info(par_wrapper);
}
}
/* MASTER - Identify unique hosts */
if (par_wrapper -> this_machine -> rank == MASTER)
{
int i, j;
for (i = 0; i < par_wrapper -> num_procs; i++)
{
if (par_wrapper -> machines[i] == (machine *)NULL)
{
continue;
}
/* All machines are initially unique */
par_wrapper -> machines[i] -> unique = 1;
}
for (i = 0; i < par_wrapper -> num_procs; i++)
{
if (par_wrapper -> machines[i] == (machine *)NULL)
{
continue;
}
if (par_wrapper -> machines[i] -> unique == 0)
{
continue; /* This host is already not unique */
}
for (j = i + 1; j < par_wrapper -> num_procs; j++)
{
if (par_wrapper -> machines[j] == (machine *)NULL)
{
continue;
}
if (par_wrapper -> machines[j] -> unique == 0)
{
continue; /* This host is already not unique */
}
if (strcmp(par_wrapper -> machines[i] -> ip_addr, par_wrapper -> machines[j] -> ip_addr) == 0)
{
par_wrapper -> machines[j] -> unique = 0;
debug(PRNT_INFO, "Rank %d (%s:%d) is not unique (some host as %d).\n",
j, par_wrapper -> machines[j] -> ip_addr,
par_wrapper -> machines[j] -> port, i);
}
}
}
}
int shared_fs = 1; /* Flag which denotes a shared fs */
/* If I am rank 0 - determine if we need to create a fake file system */
if (par_wrapper -> this_machine -> rank == MASTER)
{
int i;
for (i = 0; i < par_wrapper -> num_procs; i++)
{
if (par_wrapper -> machines[i] == (machine *)NULL)
{
continue;
}
if (strcmp(par_wrapper -> machines[i] -> iwd, par_wrapper -> master -> iwd) != 0)
{
shared_fs = 0;
}
}
if (shared_fs == 0)
{
char fake_fs[1024];
time_t curr_time = time(NULL);
snprintf(fake_fs, 1024, "/tmp/condor_hydra_%d_%ld", par_wrapper -> cluster_id,
(long)curr_time);
debug(PRNT_INFO, "Using fake file system (%s). IWD's across ranks differ\n", fake_fs);
for (i = 0; i < par_wrapper -> num_procs; i++)
{
if (par_wrapper -> machines[i] == (machine *)NULL)
{
continue;
}
if (! par_wrapper -> machines[i] -> unique)
{
continue;
}
/* Send the command to create softlinks */
struct timeval old_time = par_wrapper -> machines[i] -> last_alive;
while ( 1 )
{
RC = create_link(par_wrapper -> command_socket, par_wrapper -> machines[i] -> iwd,
fake_fs, par_wrapper -> machines[i] -> ip_addr,
par_wrapper -> machines[i] -> port);
usleep(100000); /* Sleep for 1/10th of a second */
if (RC == 0 &&
((old_time.tv_sec != par_wrapper -> machines[i] -> last_alive.tv_sec) ||
(old_time.tv_usec != par_wrapper -> machines[i] -> last_alive.tv_usec)))
{
break;
}
debug(PRNT_INFO, "Waiting for ACK from rank %d\n", i);
}
}
par_wrapper -> shared_fs = strdup(fake_fs);
}
else
{
par_wrapper -> shared_fs = strdup(par_wrapper -> master -> iwd);
debug(PRNT_INFO, "Using a shared file system, IWD = %s\n", par_wrapper -> master -> iwd);
}
}
/* Unlock the keepalive mutex */
pthread_mutex_unlock(&keep_alive_mutex);
/* Start up the MPI executable */
if (par_wrapper -> this_machine -> rank == MASTER)
{
par_wrapper -> child_pid = fork();
if (par_wrapper -> child_pid == (pid_t) -1)
{
print(PRNT_ERR, "Fork failed\n");
cleanup(par_wrapper, 10);
}
else if (par_wrapper -> child_pid == (pid_t) 0)
{
/* I am the child */
prctl(PR_SET_PDEATHSIG, SIGTERM);
/* Set environment variables */
char *machine_file = join_paths(par_wrapper -> scratch_dir, MACHINE_FILE);
char *ssh_config = join_paths(par_wrapper -> scratch_dir, SSH_CONFIG);
setenv("MACHINE_FILE", machine_file, 1);
setenv("SSH_CONFIG", ssh_config, 1);
free(machine_file);
free(ssh_config);
char *ssh_wrapper = join_paths(par_wrapper -> scratch_dir, SSH_WRAPPER);
setenv("SSH_WRAPPER", ssh_wrapper, 1);
free(ssh_wrapper);
char temp_str[1024];
snprintf(temp_str, 1024, "%d", par_wrapper -> num_procs);
setenv("NUM_MACHINES", temp_str, 1);
/* Determine the total number of cpus allocated to this task */
int total_cpus = 0;
int i;
for (i = 0; i < par_wrapper -> num_procs; i++)
{
if (par_wrapper -> machines[i] == (machine *)NULL)
{
continue;
}
total_cpus += par_wrapper -> machines[i] -> cpus;
}
snprintf(temp_str, 1024, "%d", total_cpus);
setenv("CPUS", temp_str, 1);
setenv("NUM_PROCS", temp_str, 1);
snprintf(temp_str, 1024, "%d", par_wrapper -> this_machine -> rank);
setenv("RANK", temp_str, 1);
snprintf(temp_str, 1024, "%d", par_wrapper -> cluster_id);
setenv("CLUSTER_ID", temp_str, 1);
snprintf(temp_str, 1024, "%d", par_wrapper -> this_machine -> port);
setenv("CMD_PORT", temp_str, 1);
snprintf(temp_str, 1024, "%d", par_wrapper -> this_machine -> cpus);
setenv("REQUEST_CPUS", temp_str, 1);
setenv("SCRATCH_DIR", par_wrapper -> scratch_dir, 1);
setenv("IWD", par_wrapper -> this_machine -> iwd, 1);
setenv("IP_ADDR", par_wrapper -> this_machine -> ip_addr, 1);
setenv("TRANSFER_FILES", shared_fs != 0 ? "TRUE" : "FALSE", 1);
setenv("SHARED_FS", shared_fs != 0 ? par_wrapper -> this_machine -> iwd : par_wrapper -> shared_fs, 1);
setenv("SHARED_DIR", shared_fs != 0 ? par_wrapper -> this_machine -> iwd : par_wrapper -> shared_fs, 1);
setenv("SCHEDD_IWD", par_wrapper -> this_machine -> schedd_iwd, 1);
/* TODO: SSH ENVS */
/* Search in path */
int process_RC = execvp(par_wrapper -> executable[0], &par_wrapper -> executable[0]);
if (process_RC != 0)
{
print(PRNT_ERR, "%s\n", get_exec_error_msg(errno, par_wrapper -> executable[0]));
}
exit(process_RC);
}
else
{
/* I am the parent */
int child_status = 0;
/* Create a new group for the child processes */
par_wrapper -> pgid = setpgid(par_wrapper -> child_pid,
par_wrapper -> child_pid);
if (par_wrapper -> pgid < 0)
{
print(PRNT_WARN, "Unable to set process group for children\n");
}
waitpid(par_wrapper -> child_pid, &child_status, WUNTRACED); /* Wait for the child to finish */
cleanup(par_wrapper, child_status);
}
}
/* Always wait for the listener */
pthread_join(par_wrapper -> listener, NULL);
return 0;
}
GPUCommands()
: m_cmd() {
register_cmd(new GpuCommand_list_free_mem);
register_cmd(new GpuCommand_list_num_registers);
register_cmd(new GpuCommand_list_free_registers);
register_cmd(new GpuCommand_list_used_registers);
register_cmd(new GpuCommand_describe_register);
register_cmd(new GpuCommand_print_reg_data);
register_cmd(new GpuCommand_load);
register_cmd(new GpuCommand_store);
register_cmd(new GpuCommand_add_fld);
register_cmd(new GpuCommand_del_fld);
register_cmd(new GpuCommand_swap_flds);
register_cmd(new GpuCommand_f1s1opf2);
register_cmd(new GpuCommand_f1f2opf3);
register_cmd(new GpuCommand_f_to_s);
register_cmd(new GpuCommand_f1f2_to_s);
register_cmd(new GpuCommand_f1opf2);
register_cmd(new GpuCommand_f1_shift_f2);
register_cmd(new GpuCommand_count);
register_cmd(new GpuCommand_count_f);
register_cmd(new GpuCommand_join);
register_cmd(new GpuCommand_fop);
register_cmd(new GpuCommand_sort);
register_cmd(new GpuCommand_mk_idx);
register_cmd(new GpuCommand_permute_by_idx);
register_cmd(new GpuCommand_funnel);
register_cmd(new GpuCommand_funnel_count);
register_cmd(new GpuCommand_sortf1f2);
register_cmd(new GpuCommand_print_data_to_ascii_file);
}
static void init_module(acetables *g_ape) // Called when module is loaded
{
register_cmd("CONTROL", 5, cmd_control, NEED_NOTHING, g_ape);
}
/**
* Initializes the list of firmware commands in a pointer table
*/
void register_cmds() {
// help
register_cmd("HELP" ,cmd_help);
// log
register_cmd("LOGXFER" ,cmd_logxfer);
register_cmd("READ CSV LOG" ,cmd_logcsv);
register_cmd("LOGSIG" ,cmd_logsig);
register_cmd("LOGPAUSE" ,cmd_logpause);
register_cmd("LOGRESUME" ,cmd_logresume);
register_cmd("LOGCLEAR" ,cmd_logclear);
// get
register_cmd("VERSION" ,cmd_version); // GETVERSION
register_cmd("GUID" ,cmd_guid); // GETGUID
register_cmd("GETDEVICETAG" ,cmd_getdevicetag);
register_cmd("MAGREAD" ,cmd_magread); // GETMAG ? hall sensor?
// cpm
register_cmd("GETCPM" ,cmd_cpm); // CPM
// ??
register_cmd("WRITEDAC" ,cmd_writedac);
register_cmd("READADC" ,cmd_readadc);
// set
register_cmd("SETDEVICETAG" ,cmd_setdevicetag);
register_cmd("SETMICREVERSE" ,cmd_setmicreverse);
register_cmd("SETMICIPHONE" ,cmd_setmiciphone);
register_cmd("DISPLAYPARAMS" ,cmd_displayparams); // SETDIAPLAYPARAMS
register_cmd("SETRTC" ,cmd_setrtc);
register_cmd("SETALARM" ,cmd_setalarm);
// test/debug
register_cmd("DISPLAYTEST" ,cmd_displaytest); // TESTDISPLAY
register_cmd("BATINFODISP" ,cmd_batinfodisp); // TESTBATDISPLAY
register_cmd("TESTHP" ,cmd_testhp); // ??
register_cmd("LOGSTRESS" ,cmd_logstress); // TESTLOG
register_cmd("TESTSIGN" ,cmd_testsign);
register_cmd("PUBKEY" ,cmd_pubkey);
register_cmd("KEYVALID" ,cmd_keyvalid);
register_cmd("KEYVALDUMP" ,cmd_keyvaldump);
register_cmd("KEYVALSET" ,cmd_keyvalset);
register_cmd("CAPTOUCHTEST" ,cmd_captouchparams);
register_cmd("CAPTOUCHDUMP" ,cmd_captouchdump);
// misc
register_cmd("HELLO" ,cmd_hello);
register_cmd("LIST GAMES" ,cmd_games);
}
