void build_bridge(bridge_info* bridge) {
attr_list contact_list = attr_list_from_string(bridge->contact);
if(bridge->created == 0){
bridge->bridge_stone =
EVcreate_bridge_action(fp_read_data->fp_cm,
contact_list,
(EVstone)bridge->their_num);
bridge->flush_source =
EVcreate_submit_handle(fp_read_data->fp_cm,
bridge->bridge_stone,
flush_format_list);
bridge->var_source =
EVcreate_submit_handle(fp_read_data->fp_cm,
bridge->bridge_stone,
var_format_list);
bridge->op_source =
EVcreate_submit_handle(fp_read_data->fp_cm,
bridge->bridge_stone,
op_format_list);
bridge->created = 1;
}
}
int main(int argc, char *argv[])
{
/*
* We want the dfg creator file to be able to read in a list of nodes, a list of stones per node and links between stones.
* */
if(argc != 2)
usage();
else
{
std::vector<std::string> type_names;
std::string config_file_name = argv[1];
ConfigParser_t cfg;
if(cfg.readFile(config_file_name))
{
printf("Error: Cannot open config file %s", config_file_name.c_str());
return 1;
}
/*Read the file information into the appropriate data structures for later use*/
std::vector<std::string> stone_sections = cfg.getSections();
std::vector<std::string> temp_node_name_list;
for(std::vector<std::string>::iterator I = stone_sections.begin(), E = stone_sections.end(); I != E; ++I)
{
stone_struct new_stone_struct;
stone_struct pot_python_struct;
/*Get the node name for the stone and in doing so,
bookkeep which node names that we have already seen*/
std::string which_node;
if(!config_read_node(cfg, *I, which_node))
{
fprintf(stderr, "Failure to read node from config\n");
exit(1);
}
unsigned int temp_i = 0;
for(; temp_i < temp_node_name_list.size(); ++temp_i)
{
if(!which_node.compare(temp_node_name_list[temp_i]))
break;
}
if(temp_i == temp_node_name_list.size())
{
temp_node_name_list.push_back(which_node);
++test_dfg.node_count;
}
new_stone_struct.node_name = which_node;
/*Read the stone type into the enum value...*/
if(!config_read_type(cfg, *I, new_stone_struct.stone_type))
{
fprintf(stderr, "Error: reading config type of stone failed\n");
exit(1);
}
/*Store the section name as the stone name
This must be a unique value for every stone*/
if(new_stone_struct.stone_type == PYTHON)
{
if(!setupPythonStones(cfg, *I, new_stone_struct, pot_python_struct))
{
log_err("Error: failed to setup python stones");
exit(1);
}
stone_holder.push_back(new_stone_struct);
stone_holder.push_back(pot_python_struct);
}
else if(new_stone_struct.stone_type == BUCKETROLL)
{
if(!setupBucketStone(cfg, *I, new_stone_struct))
{
log_err("Error: failed to setup bucket stone");
exit(1);
}
stone_holder.push_back(new_stone_struct);
}
else
{
new_stone_struct.stone_name = *I;
/*Construct a unique handler for each source and sink node
For now the nodes will have to read the config file*/
std::string unique_handler_name = new_stone_struct.stone_name + "_" + new_stone_struct.node_name;
new_stone_struct.src_sink_handler_name = unique_handler_name;
/*Read the incoming stones to connect to from the config file*/
if(!config_read_incoming(cfg, *I, new_stone_struct.incoming_stones))
{
log_err("Error reading incoming stones");
exit(1);
}
/*TODO:Read the actual code, probably going to need to
have a seperate code file for the COD code*/
//Push-back here
stone_holder.push_back(new_stone_struct);
/*
printf("Testing: node_name := %s\n", new_stone_struct.node_name.c_str());
printf("Testing: stone_name := %s\n", new_stone_struct.stone_name.c_str());
printf("Testing: handler_name := %s\n", new_stone_struct.src_sink_handler_name.c_str());
printf("Testing: stone_type := %d\n", new_stone_struct.stone_type);
printf("Testing: node_count := %d\n", test_dfg.node_count);
for(unsigned int i = 0; i < new_stone_struct.incoming_stones.size(); ++i)
{
printf("Testing: incoming_stone for %s: %s\n", new_stone_struct.stone_name.c_str(),
new_stone_struct.incoming_stones[i].c_str());
}
printf("Testing: code_type := %d\n", new_stone_struct.code_type);
*/
}
}
// Master node entry stone added to the end of the vector
stone_struct entry_stone_struct;
entry_stone_struct.node_name = "master_node";
entry_stone_struct.stone_name = "entry";
entry_stone_struct.src_sink_handler_name = entry_stone_struct.stone_name + "_" + entry_stone_struct.node_name;
entry_stone_struct.stone_type = SOURCE;
stone_holder.push_back(entry_stone_struct);
++test_dfg.node_count;
if(dfg_init_func())
{
EVmaster_node_join_handler(test_dfg.dfg_master, JoinHandlerFunc);
printf("DFG handler read...\n");
}
/*EVsource * master_list_sources = (EVsource *) malloc(sizeof(EVsource) * temp_node_name_list.size());
for(unsigned int i = 0; i < temp_node_name_list.size(); ++i)
{
//Place the list stones into the list
stone_struct py_master_list_source;
py_master_list_source.node_name = "master_node";
py_master_list_source.stone_name = "to_" + temp_node_name_list[i];
py_master_list_source.src_sink_handler_name = py_master_list_source.stone_name + "_" + py_master_list_source.node_name;
py_master_list_source.stone_type = SOURCE;
stone_holder.push_back(py_master_list_source);
//Register the handler
master_list_sources[i] = EVcreate_submit_handle(test_dfg.cm, -1, python_format_list);
char * perm_ptr = strdup(py_master_list_source.src_sink_handler_name.c_str());
source_capabilities = EVclient_register_source(perm_ptr, master_list_sources[i]);
stone_struct py_master_list_sink;
py_master_list_sink.node_name = temp_node_name_list[i];
py_master_list_sink.stone_name = "python_list";
py_master_list_sink.src_sink_handler_name = py_master_list_sink.stone_name + "_" + py_master_list_sink.node_name;
py_master_list_sink.incoming_stones.push_back(py_master_list_source.stone_name);
py_master_list_sink.stone_type = SINK;
stone_holder.push_back(py_master_list_sink);
}
*/
EVclient_sources source_capabilities;
EVsource source_handle;
EVclient master_client;
source_handle = EVcreate_submit_handle(test_dfg.cm, -1, simple_format_list);
char * temp_char_ptr = strdup(entry_stone_struct.src_sink_handler_name.c_str());
source_capabilities = EVclient_register_source(temp_char_ptr, source_handle);
temp_char_ptr = NULL;
master_client = EVclient_assoc_local(test_dfg.cm, "master_node", test_dfg.dfg_master, source_capabilities, NULL);
if (EVclient_ready_wait(master_client) != 1) {
/* initialization failed! */
log_err("EVclient_ready_wait: FAILED!");
exit(1);
}
/*
python_list test_py_list;
test_py_list.dynamic_size = 4;
test_py_list.ordered_method_list[0] = 0;
test_py_list.ordered_method_list[1] = 1;
test_py_list.ordered_method_list[2] = 2;
test_py_list.ordered_method_list[3] = 3;
for(int i = 0; i < temp_node_name_list.size(); ++i)
{
EVsubmit(master_list_sources[i], &test_py_list, NULL);
printf("Submitted a python list...\n");
}
*/
/* Setting up the options and stuff, before sending anything */
std::string store_filename="stores.txt"; /* Default */
int port = 6379;
int opt;
std::string host = "localhost";
while ((opt = getopt (argc, argv, "p:s:h:")) != -1)
{
switch (opt)
{
case 'p':
port = atoi(optarg);
break;
case 's':
store_filename = optarg;
break;
case 'h':
host = optarg;
break;
}
}
FILE* stores = fopen(store_filename.c_str(), "r");
char buffer[100];
char buffer2[100];
fscanf(stores, "%s\n", buffer);
this_server_id = buffer;
while(fscanf(stores, "%s %s\n", buffer, buffer2)!=EOF) {
servers.push_back(buffer);
server_ids.push_back(buffer2);
if(this_server_id == buffer2) {
this_server = buffer;
}
}
fclose(stores);
/* Initialization of Khan */
arg_struct khan_args;
khan_args.mnt_dir = argv[1];
khan_args.servers = servers;
khan_args.server_ids = server_ids;
khan_args.port = port;
khan_args.host = host;
initializing_khan((void*)&khan_args);
log_info("Initialized Khan");
// Hardcode the im7 type as the only type for now
type_names.push_back("im7");
initialize_attrs_for_data_types(type_names);
/* Here's where we set up and send the data */
simple_rec data;
std::string pattern = servers[0] + "/*";
glob_t files;
int fdin;
struct stat statbuf;
std::set<std::string> experiments;
for(int count = 18; count > 0; count--) {
glob((pattern +".im7").c_str(), 0, NULL, &files);
log_info("Globbing with pattern: %s.im7", pattern.c_str());
for(unsigned j=0; j<files.gl_pathc; j++) {
std::string filepath = files.gl_pathv[j];
log_info("File Path: %s", filepath.c_str());
std::string exp_dir = filepath.substr(0, filepath.size() - 10);
experiments.insert(exp_dir);
data.exp_id = (int)experiments.size();
data.file_path = strdup(filepath.c_str());
if ((fdin = open (filepath.c_str(), O_RDONLY)) < 0)
perror ("can't open %s for reading");
if (fstat (fdin,&statbuf) < 0)
perror ("fstat error");
data.file_buf_len = statbuf.st_size;
log_info("Size: %zu", data.file_buf_len);
if ((data.file_buf = (char*)mmap (0, statbuf.st_size, PROT_READ, MAP_PRIVATE, fdin, 0))
== (caddr_t) -1)
perror ("mmap error for input");
/* Adding in code here to ensure that the event gets setup in Redis correctly before it gets sent out */
//TODO:Delete the bottom two lines if I've compiled successfully without them
//std::string original_fp (data.file_path);
//std::string shared_fn = "/dev/shm/" + original_fp.substr(49, strlen(data.file_path) - 49);
if(!init_database_values(data))
{
log_err("Error in initializing database values for %s", data.file_path);
exit(1);
}
//Set Metadata value to 0
data.meta_compare_py = 0;
EVsubmit(source_handle, &data, NULL);
if (munmap(data.file_buf, statbuf.st_size) == -1) {
perror("Error un-mmapping the file");
}
/*Free Memory*/
close(fdin);
free(data.file_path);
}
pattern += "/*";
}
log_info("Cleanup data structures");
/* Cleanup */
globfree(&files);
log_info("Shutdown evdfg");
CMrun_network(test_dfg.cm);
}
return 0;
}
int
adios_read_icee_init_method (MPI_Comm comm, PairStruct* params)
{
log_debug ("%s\n", __FUNCTION__);
int cm_port = 59997;
char *cm_host = "localhost";
int cm_remote_port = 59999;
char *cm_remote_host = "localhost";
char *cm_attr = NULL;
//attr_list contact_list;
icee_transport_t icee_transport_init = TCP;
icee_transport_t icee_transport = TCP;
icee_contactinfo_rec_t *remote_contact = NULL;
int i;
int use_single_remote_server = 1;
char *remote_list_str = NULL;
char *attr_list_str = NULL;
int use_native_contact = 0;
PairStruct * p = params;
while (p)
{
if (!strcasecmp (p->name, "cm_attr"))
{
cm_attr = p->value;
}
else if (!strcasecmp (p->name, "cm_host"))
{
cm_host = p->value;
}
else if (!strcasecmp (p->name, "cm_port"))
{
cm_port = atoi(p->value);
}
else if (!strcasecmp (p->name, "cm_remote_host"))
{
cm_remote_host = p->value;
}
else if (!strcasecmp (p->name, "cm_remote_port"))
{
cm_remote_port = atoi(p->value);
}
else if (!strcasecmp (p->name, "remote_list"))
{
use_single_remote_server = 0;
if (p->value)
remote_list_str = strdup(p->value);
}
else if (!strcasecmp (p->name, "attr_list"))
{
use_single_remote_server = 0;
if (p->value)
attr_list_str = strdup(p->value);
}
else if (!strcasecmp (p->name, "transport"))
{
if (strcasecmp(p->value, "TCP") == 0)
icee_transport = TCP;
else if (strcasecmp(p->value, "ENET") == 0)
icee_transport = ENET;
else if (strcasecmp(p->value, "NNTI") == 0)
icee_transport = NNTI;
else if (strcasecmp(p->value, "IB") == 0)
icee_transport = IB;
else
log_error ("No support: %s\n", p->value);
}
else if (!strcasecmp (p->name, "transport_init"))
{
if (strcasecmp(p->value, "TCP") == 0)
icee_transport_init = TCP;
else if (strcasecmp(p->value, "ENET") == 0)
icee_transport_init = ENET;
else if (strcasecmp(p->value, "NNTI") == 0)
icee_transport_init = NNTI;
else if (strcasecmp(p->value, "IB") == 0)
icee_transport_init = IB;
else
log_error ("No support: %s\n", p->value);
}
else if (!strcasecmp (p->name, "num_parallel"))
{
icee_read_num_parallel = atoi(p->value);
}
else if (!strcasecmp (p->name, "is_passive"))
{
is_read_cm_passive = atoi(p->value);
}
else if (!strcasecmp (p->name, "use_native_contact"))
{
use_native_contact = atoi(p->value);
}
p = p->next;
}
pthread_mutex_init(&fileinfo_lock, NULL);
if (use_single_remote_server)
{
num_remote_server = 1;
attr_list contact_list = create_attr_list();
set_contact_list(contact_list, icee_transport_init, cm_remote_host, cm_remote_port);
icee_contactinfo_rec_t *p;
p = malloc(sizeof(icee_contactinfo_rec_t));
char *contact_string = attr_list_to_string(contact_list);
p->contact_string = contact_string;
p->stone_id = 0; // we assume. it can be wrong.
p->next = NULL;
remote_contact = p;
}
else
{
num_remote_server = 0;
icee_contactinfo_rec_t *p;
icee_contactinfo_rec_t *prev;
char* token = strtok(remote_list_str, ",");
while (token)
{
char host[256];
int port = 0;
if (token[0] == ':')
{
strcpy(host, cm_remote_host);
port = atoi(token+1);
}
else
{
char *pch = strchr(token, ':');
if (pch != NULL)
{
strncpy(host, token, pch - token);
host[pch-token] = '\0';
port = atoi(pch+1);
}
else
{
int len = strlen(token);
strncpy(host, token, len);
assert(len < 256);
host[len] = '\0';
port = cm_remote_port;
}
}
log_debug("Remote server list: (%d) %s:%d\n", num_remote_server, host, port);
p = malloc(sizeof(icee_contactinfo_rec_t));
attr_list contact_list;
contact_list = create_attr_list();
set_contact_list(contact_list, icee_transport_init, host, port);
p->contact_string = attr_list_to_string(contact_list);
p->stone_id = 0; // we assume. it can be wrong.
p->next = NULL;
if (num_remote_server == 0)
remote_contact = p;
else
prev->next = p;
prev = p;
num_remote_server++;
token = strtok(NULL, ",");
}
}
if (attr_list_str != NULL)
{
num_remote_server = 0;
icee_contactinfo_rec_t *p;
icee_contactinfo_rec_t *prev;
char* token = strtok(attr_list_str, ",");
while (token)
{
int remote_stone = 0;
char string_list[256];
sscanf(token, "%d:%s", &remote_stone, &string_list[0]);
p = malloc(sizeof(icee_contactinfo_rec_t));
attr_list contact_list;
p->stone_id = remote_stone;
p->contact_string = strdup(string_list);
p->next = NULL;
if (num_remote_server == 0)
remote_contact = p;
else
prev->next = p;
prev = p;
num_remote_server++;
token = strtok(NULL, ",");
}
}
if (icee_read_num_parallel > ICEE_MAX_PARALLEL)
{
icee_read_num_parallel = ICEE_MAX_PARALLEL;
log_info ("Max. number of threads is set to %d\n", icee_read_num_parallel);
}
log_debug ("transport : %s\n", icee_transport_name[icee_transport]);
/*
log_info ("cm_host : %s\n", cm_host);
log_info ("cm_port : %d\n", cm_port);
for (i = 0; i < num_remote_server; i++)
{
log_info ("remote_list : %s:%d\n", remote_server[i].client_host, remote_server[i].client_port);
}
*/
if (!adios_read_icee_initialized)
{
if (is_read_cm_passive)
{
icee_contactinfo_rec_t *prev;
for (i = 0; i < num_remote_server; i++)
{
attr_list contact_list;
icee_contactinfo_rec_t *p = (i == 0)? remote_contact : prev->next;
pcm[i] = CManager_create();
if (!CMfork_comm_thread(pcm[i]))
printf("Fork of communication thread[%d] failed.\n", i);
contact_list = attr_list_from_string(p->contact_string);
log_debug("Passive remote contact: \"%s\"\n", attr_list_to_string(contact_list));
if (adios_verbose_level > 5) dump_attr_list(contact_list);
/*
attr_list contact_list = create_attr_list();
add_string_attr(contact_list, attr_atom_from_string("IP_HOST"),
remote_server[i].client_host);
add_int_attr(contact_list, attr_atom_from_string("IP_PORT"),
remote_server[i].client_port);
*/
CMConnection conn = CMinitiate_conn(pcm[i], contact_list);
int n = 0;
while (conn == NULL)
{
log_error ("Passive connection failed (%d). Try again ...\n", i);
dump_attr_list(contact_list);
sleep(2);
conn = CMinitiate_conn(pcm[i], contact_list);
if (n > 5) break;
n++;
}
if (conn == NULL)
{
log_error ("Initializing passive connection failed (%d)\n", i);
}
CMFormat fm_checkin, fm_fileinfo;
fm_checkin = CMregister_format(pcm[i], icee_passivecheckin_format_list);
CMregister_handler(fm_checkin, icee_passivecheckin_reply_handler, on_icee_passivecheckin_reply);
fm_fileinfo = CMregister_format(pcm[i], icee_fileinfo_format_list);
CMregister_handler(fm_fileinfo, icee_fileinfo_recv_handler, on_icee_fileinfo_recv);
icee_passivecheckin_rec_t m;
int condition = CMCondition_get(pcm[i], conn);
CMCondition_set_client_data(pcm[i], condition, NULL);
m.condition = condition;
if (CMwrite(conn, fm_checkin, (void*)&m) != 1)
log_error ("Passive check-in failed (%d)\n", i);
prev = p;
}
log_debug("Passive connection established");
goto done;
}
EVstone stone[ICEE_MAX_PARALLEL], remote_stone;
EVsource source;
attr_list contact[ICEE_MAX_PARALLEL];
icee_contactinfo_rec_t contact_msg[ICEE_MAX_PARALLEL];
for (i=0; i<icee_read_num_parallel; i++)
{
icee_read_cm[i] = CManager_create();
contact[i] = create_attr_list();
set_contact_list(contact[i], icee_transport, cm_host, cm_port+i);
if (CMlisten_specific(icee_read_cm[i], contact[i]) == 0)
printf("Error: unable to initialize connection manager[%d].\n", i);
if (!CMfork_comm_thread(icee_read_cm[i]))
printf("Fork of communication thread[%d] failed.\n", i);
stone[i] = EValloc_stone(icee_read_cm[i]);
if (adios_verbose_level > 5)
{
log_debug("Reader contact: \"%d:%s\"\n", stone[i], attr_list_to_string(CMget_contact_list(icee_read_cm[i])));
dump_attr_list(CMget_contact_list(icee_read_cm[i]));
}
EVassoc_terminal_action(icee_read_cm[i], stone[i], icee_fileinfo_format_list, icee_fileinfo_handler, NULL);
contact_msg[i].stone_id = stone[i];
attr_list contact_list;
if (use_native_contact)
contact_list = CMget_contact_list(icee_read_cm[i]);
else
contact_list = contact[i];
contact_msg[i].contact_string = attr_list_to_string(contact_list);
contact_msg[i].next = NULL;
if (i>0)
contact_msg[i-1].next = &contact_msg[i];
}
EVstone split_stone;
EVaction split_action;
split_stone = EValloc_stone(icee_read_cm[0]);
split_action = EVassoc_split_action(icee_read_cm[0], split_stone, NULL);
icee_contactinfo_rec_t *prev;
for (i = 0; i < num_remote_server; i++)
{
attr_list contact_list;
EVstone remote_stone, output_stone;
output_stone = EValloc_stone(icee_read_cm[0]);
icee_contactinfo_rec_t *p = (i == 0)? remote_contact : prev->next;
remote_stone = p->stone_id;
contact_list = attr_list_from_string(p->contact_string);
EVaction action;
action = EVassoc_bridge_action(icee_read_cm[0], output_stone, contact_list, remote_stone);
int n = 0;
while (action == -1)
{
log_error ("Connection failed (%d). Try again ...\n", i);
dump_attr_list(contact_list);
sleep(2);
action = EVassoc_bridge_action(icee_read_cm[0], output_stone, contact_list, remote_stone);
if (n > 5) break;
n++;
}
EVaction_add_split_target(icee_read_cm[0], split_stone, split_action, output_stone);
prev = p;
log_debug("Remote contact: \"%d:%s\"\n", remote_stone, attr_list_to_string(contact_list));
if (adios_verbose_level > 5) dump_attr_list(contact_list);
}
source = EVcreate_submit_handle(icee_read_cm[0], split_stone, icee_contactinfo_format_list);
//if (adios_verbose_level > 5) icee_contactinfo_print(contact_msg);
EVsubmit(source, contact_msg, NULL);
done:
adios_read_icee_initialized = 1;
}
return 0;
}
int main(int argc, char **argv)
{
struct sigaction act;
memset (&act, '\0', sizeof(act));
act.sa_sigaction = &cleanup_handler;
act.sa_flags = 0;
measurements_init();
EVclient test_client;
EVclient_sinks sink_capabilities;
EVclient_sources source_capabilities;
(void)argc; (void)argv;
cm = CManager_create();
CMlisten(cm);
char master_address[200];
dfg_get_master_contact_func(master_address,"master.info");
char source_node[300] = "src_";
strcat(source_node, argv[1]);
stor_source_handle = EVcreate_submit_handle(cm, -1, simple_format_list);
source_capabilities = EVclient_register_source(source_node, stor_source_handle);
sink_capabilities = EVclient_register_sink_handler(cm, "sink_b", simple_format_list,
(EVSimpleHandlerFunc) simple_handler, NULL);
/* We're node "a" in the DFG */
test_client = EVclient_assoc(cm, argv[1], master_address, source_capabilities, sink_capabilities);
if (EVclient_ready_wait(test_client) != 1) {
/* dfg initialization failed! */
exit(1);
}
/**************************/
if (sigaction(SIGINT, &act, NULL) < 0) {
perror ("sigaction");
return 1;
}
//signal(SIGUSR1, create_graphs);
Py_SetProgramName(argv[0]); /* optional but recommended */
Py_Initialize();
PyObject *sys = PyImport_ImportModule("sys");
PyObject *path = PyObject_GetAttrString(sys, "path");
char cwd[1024];
std::string py = "/PyScripts";
std::string pyscripts_path = strdup(getcwd(cwd, sizeof(cwd))) + py;
PyList_Append(path, PyString_FromString(pyscripts_path.c_str()));
PySys_SetObject("path", path);
std::string store_filename="stores.txt"; /* Default */
int port = 6379;
int opt;
std::string host = "localhost";
while ((opt = getopt (argc, argv, "p:s:h:")) != -1)
{
switch (opt)
{
case 'p':
port = atoi(optarg);
break;
case 's':
store_filename = optarg;
break;
case 'h':
host = optarg;
break;
}
}
/* Set signal handler */
//signal(SIGTERM, cleanup_handler);
//signal(SIGKILL, cleanup_handler);
//signal(SIGSEGV, cleanup_handler);
FILE* stores = fopen(store_filename.c_str(), "r");
char buffer[100];
char buffer2[100];
fscanf(stores, "%s\n", buffer);
this_server_id = buffer;
while(fscanf(stores, "%s %s\n", buffer, buffer2)!=EOF) {
servers.push_back(buffer);
server_ids.push_back(buffer2);
if(this_server_id == buffer2) {
this_server = buffer;
}
}
fclose(stores);
arg_struct khan_args;
khan_args.mnt_dir = argv[1];
khan_args.servers = servers;
khan_args.server_ids = server_ids;
khan_args.port = port;
khan_args.host = host;
initializing_khan((void*)&khan_args);
log_info("Initialized Khan");
CMrun_network(cm);
return 0;
}
