int main(int argc, char *argv[])
{
/*
* We want the dfg creator file to be able to read in a list of nodes, a list of stones per node and links between stones.
* */
if(argc != 2)
usage();
else
{
std::vector<std::string> type_names;
std::string config_file_name = argv[1];
ConfigParser_t cfg;
if(cfg.readFile(config_file_name))
{
printf("Error: Cannot open config file %s", config_file_name.c_str());
return 1;
}
/*Read the file information into the appropriate data structures for later use*/
std::vector<std::string> stone_sections = cfg.getSections();
std::vector<std::string> temp_node_name_list;
for(std::vector<std::string>::iterator I = stone_sections.begin(), E = stone_sections.end(); I != E; ++I)
{
stone_struct new_stone_struct;
stone_struct pot_python_struct;
/*Get the node name for the stone and in doing so,
bookkeep which node names that we have already seen*/
std::string which_node;
if(!config_read_node(cfg, *I, which_node))
{
fprintf(stderr, "Failure to read node from config\n");
exit(1);
}
unsigned int temp_i = 0;
for(; temp_i < temp_node_name_list.size(); ++temp_i)
{
if(!which_node.compare(temp_node_name_list[temp_i]))
break;
}
if(temp_i == temp_node_name_list.size())
{
temp_node_name_list.push_back(which_node);
++test_dfg.node_count;
}
new_stone_struct.node_name = which_node;
/*Read the stone type into the enum value...*/
if(!config_read_type(cfg, *I, new_stone_struct.stone_type))
{
fprintf(stderr, "Error: reading config type of stone failed\n");
exit(1);
}
/*Store the section name as the stone name
This must be a unique value for every stone*/
if(new_stone_struct.stone_type == PYTHON)
{
if(!setupPythonStones(cfg, *I, new_stone_struct, pot_python_struct))
{
log_err("Error: failed to setup python stones");
exit(1);
}
stone_holder.push_back(new_stone_struct);
stone_holder.push_back(pot_python_struct);
}
else if(new_stone_struct.stone_type == BUCKETROLL)
{
if(!setupBucketStone(cfg, *I, new_stone_struct))
{
log_err("Error: failed to setup bucket stone");
exit(1);
}
stone_holder.push_back(new_stone_struct);
}
else
{
new_stone_struct.stone_name = *I;
/*Construct a unique handler for each source and sink node
For now the nodes will have to read the config file*/
std::string unique_handler_name = new_stone_struct.stone_name + "_" + new_stone_struct.node_name;
new_stone_struct.src_sink_handler_name = unique_handler_name;
/*Read the incoming stones to connect to from the config file*/
if(!config_read_incoming(cfg, *I, new_stone_struct.incoming_stones))
{
log_err("Error reading incoming stones");
exit(1);
}
/*TODO:Read the actual code, probably going to need to
have a seperate code file for the COD code*/
//Push-back here
stone_holder.push_back(new_stone_struct);
/*
printf("Testing: node_name := %s\n", new_stone_struct.node_name.c_str());
printf("Testing: stone_name := %s\n", new_stone_struct.stone_name.c_str());
printf("Testing: handler_name := %s\n", new_stone_struct.src_sink_handler_name.c_str());
printf("Testing: stone_type := %d\n", new_stone_struct.stone_type);
printf("Testing: node_count := %d\n", test_dfg.node_count);
for(unsigned int i = 0; i < new_stone_struct.incoming_stones.size(); ++i)
{
printf("Testing: incoming_stone for %s: %s\n", new_stone_struct.stone_name.c_str(),
new_stone_struct.incoming_stones[i].c_str());
}
printf("Testing: code_type := %d\n", new_stone_struct.code_type);
*/
}
}
// Master node entry stone added to the end of the vector
stone_struct entry_stone_struct;
entry_stone_struct.node_name = "master_node";
entry_stone_struct.stone_name = "entry";
entry_stone_struct.src_sink_handler_name = entry_stone_struct.stone_name + "_" + entry_stone_struct.node_name;
entry_stone_struct.stone_type = SOURCE;
stone_holder.push_back(entry_stone_struct);
++test_dfg.node_count;
if(dfg_init_func())
{
EVmaster_node_join_handler(test_dfg.dfg_master, JoinHandlerFunc);
printf("DFG handler read...\n");
}
/*EVsource * master_list_sources = (EVsource *) malloc(sizeof(EVsource) * temp_node_name_list.size());
for(unsigned int i = 0; i < temp_node_name_list.size(); ++i)
{
//Place the list stones into the list
stone_struct py_master_list_source;
py_master_list_source.node_name = "master_node";
py_master_list_source.stone_name = "to_" + temp_node_name_list[i];
py_master_list_source.src_sink_handler_name = py_master_list_source.stone_name + "_" + py_master_list_source.node_name;
py_master_list_source.stone_type = SOURCE;
stone_holder.push_back(py_master_list_source);
//Register the handler
master_list_sources[i] = EVcreate_submit_handle(test_dfg.cm, -1, python_format_list);
char * perm_ptr = strdup(py_master_list_source.src_sink_handler_name.c_str());
source_capabilities = EVclient_register_source(perm_ptr, master_list_sources[i]);
stone_struct py_master_list_sink;
py_master_list_sink.node_name = temp_node_name_list[i];
py_master_list_sink.stone_name = "python_list";
py_master_list_sink.src_sink_handler_name = py_master_list_sink.stone_name + "_" + py_master_list_sink.node_name;
py_master_list_sink.incoming_stones.push_back(py_master_list_source.stone_name);
py_master_list_sink.stone_type = SINK;
stone_holder.push_back(py_master_list_sink);
}
*/
EVclient_sources source_capabilities;
EVsource source_handle;
EVclient master_client;
source_handle = EVcreate_submit_handle(test_dfg.cm, -1, simple_format_list);
char * temp_char_ptr = strdup(entry_stone_struct.src_sink_handler_name.c_str());
source_capabilities = EVclient_register_source(temp_char_ptr, source_handle);
temp_char_ptr = NULL;
master_client = EVclient_assoc_local(test_dfg.cm, "master_node", test_dfg.dfg_master, source_capabilities, NULL);
if (EVclient_ready_wait(master_client) != 1) {
/* initialization failed! */
log_err("EVclient_ready_wait: FAILED!");
exit(1);
}
/*
python_list test_py_list;
test_py_list.dynamic_size = 4;
test_py_list.ordered_method_list[0] = 0;
test_py_list.ordered_method_list[1] = 1;
test_py_list.ordered_method_list[2] = 2;
test_py_list.ordered_method_list[3] = 3;
for(int i = 0; i < temp_node_name_list.size(); ++i)
{
EVsubmit(master_list_sources[i], &test_py_list, NULL);
printf("Submitted a python list...\n");
}
*/
/* Setting up the options and stuff, before sending anything */
std::string store_filename="stores.txt"; /* Default */
int port = 6379;
int opt;
std::string host = "localhost";
while ((opt = getopt (argc, argv, "p:s:h:")) != -1)
{
switch (opt)
{
case 'p':
port = atoi(optarg);
break;
case 's':
store_filename = optarg;
break;
case 'h':
host = optarg;
break;
}
}
FILE* stores = fopen(store_filename.c_str(), "r");
char buffer[100];
char buffer2[100];
fscanf(stores, "%s\n", buffer);
this_server_id = buffer;
while(fscanf(stores, "%s %s\n", buffer, buffer2)!=EOF) {
servers.push_back(buffer);
server_ids.push_back(buffer2);
if(this_server_id == buffer2) {
this_server = buffer;
}
}
fclose(stores);
/* Initialization of Khan */
arg_struct khan_args;
khan_args.mnt_dir = argv[1];
khan_args.servers = servers;
khan_args.server_ids = server_ids;
khan_args.port = port;
khan_args.host = host;
initializing_khan((void*)&khan_args);
log_info("Initialized Khan");
// Hardcode the im7 type as the only type for now
type_names.push_back("im7");
initialize_attrs_for_data_types(type_names);
/* Here's where we set up and send the data */
simple_rec data;
std::string pattern = servers[0] + "/*";
glob_t files;
int fdin;
struct stat statbuf;
std::set<std::string> experiments;
for(int count = 18; count > 0; count--) {
glob((pattern +".im7").c_str(), 0, NULL, &files);
log_info("Globbing with pattern: %s.im7", pattern.c_str());
for(unsigned j=0; j<files.gl_pathc; j++) {
std::string filepath = files.gl_pathv[j];
log_info("File Path: %s", filepath.c_str());
std::string exp_dir = filepath.substr(0, filepath.size() - 10);
experiments.insert(exp_dir);
data.exp_id = (int)experiments.size();
data.file_path = strdup(filepath.c_str());
if ((fdin = open (filepath.c_str(), O_RDONLY)) < 0)
perror ("can't open %s for reading");
if (fstat (fdin,&statbuf) < 0)
perror ("fstat error");
data.file_buf_len = statbuf.st_size;
log_info("Size: %zu", data.file_buf_len);
if ((data.file_buf = (char*)mmap (0, statbuf.st_size, PROT_READ, MAP_PRIVATE, fdin, 0))
== (caddr_t) -1)
perror ("mmap error for input");
/* Adding in code here to ensure that the event gets setup in Redis correctly before it gets sent out */
//TODO:Delete the bottom two lines if I've compiled successfully without them
//std::string original_fp (data.file_path);
//std::string shared_fn = "/dev/shm/" + original_fp.substr(49, strlen(data.file_path) - 49);
if(!init_database_values(data))
{
log_err("Error in initializing database values for %s", data.file_path);
exit(1);
}
//Set Metadata value to 0
data.meta_compare_py = 0;
EVsubmit(source_handle, &data, NULL);
if (munmap(data.file_buf, statbuf.st_size) == -1) {
perror("Error un-mmapping the file");
}
/*Free Memory*/
close(fdin);
free(data.file_path);
}
pattern += "/*";
}
log_info("Cleanup data structures");
/* Cleanup */
globfree(&files);
log_info("Shutdown evdfg");
CMrun_network(test_dfg.cm);
}
return 0;
}
int main(int argc, char *argv[])
{
khan_ops.getattr = khan_getattr;
khan_ops.init = khan_init;
khan_ops.access = xmp_access;
khan_ops.readlink = xmp_readlink;
khan_ops.readdir = xmp_readdir;
khan_ops.mknod = xmp_mknod;
khan_ops.mkdir = xmp_mkdir;
khan_ops.symlink = xmp_symlink;
khan_ops.unlink = xmp_unlink;
khan_ops.rmdir = xmp_rmdir;
khan_ops.rename = xmp_rename;
khan_ops.link = xmp_link;
khan_ops.chmod = xmp_chmod;
khan_ops.chown = xmp_chown;
khan_ops.truncate = xmp_truncate;
khan_ops.create = khan_create;
khan_ops.utimens = xmp_utimens;
khan_ops.open = khan_open;
khan_ops.read = xmp_read;
khan_ops.write = xmp_write;
khan_ops.statfs = xmp_statfs;
khan_ops.release = xmp_release;
khan_ops.fsync = xmp_fsync;
khan_ops.opendir = khan_opendir;
khan_ops.flush = khan_flush;
khan_ops.getxattr = xmp_getxattr;
#ifdef APPLE
khan_ops.setxattr = xmp_setxattr;
khan_ops.listxattr = xmp_listxattr;
khan_ops.removexattr = xmp_removexattr;
khan_ops.setvolname = xmp_setvolname;
khan_ops.exchange = xmp_exchange;
khan_ops.getxtimes = xmp_getxtimes;
khan_ops.setbkuptime = xmp_setbkuptime;
khan_ops.setchgtime = xmp_setchgtime;
khan_ops.setcrtime = xmp_setcrtime;
khan_ops.chflags = xmp_chflags;
khan_ops.setattr_x = xmp_setattr_x;
khan_ops.fsetattr_x = xmp_fsetattr_x;
#endif
Py_SetProgramName(argv[0]); /* optional but recommended */
Py_Initialize();
PyObject *sys = PyImport_ImportModule("sys");
PyObject *path = PyObject_GetAttrString(sys, "path");
PyList_Append(path, PyString_FromString("."));
int retval=0;
struct khan_param param = { 0, 0, NULL, 0 };
if((argc<2)||(argc>4)) {
printf("Usage: ./khan <mount_dir_location> [stores.txt] [-d]\nAborting...\n");
exit(1);
}
struct fuse_args args = FUSE_ARGS_INIT(0, NULL);
int j;
const char* store_filename="stores.txt";
for(j = 0; j < argc; j++) {
if((j == 2) && (argv[j][0]!='-')) {
store_filename = argv[j];
} else {
fuse_opt_add_arg(&args, argv[j]);
}
}
//set signal handler
signal(SIGTERM, my_terminate);
signal(SIGKILL, my_terminate);
fprintf(stderr, "store filename: %s\n", store_filename);
FILE* stores = fopen(store_filename, "r");
char buffer[100];
char buffer2[100];
fscanf(stores, "%s\n", buffer);
this_server_id = buffer;
while(fscanf(stores, "%s %s\n", buffer, buffer2)!=EOF) {
if(strcmp(buffer,"cloud")==0) {
string module = buffer2;
module = "cloud." + module;
cloud_interface = PyImport_ImportModule(module.c_str());
}
servers.push_back(buffer);
server_ids.push_back(buffer2);
if(this_server_id == buffer2) {
this_server = buffer;
}
}
fclose(stores);
umask(0);
if(-1==log_open()) {
printf("Unable to open the log file..NO log would be recorded..!\n");
}
//log_msg("\n\n--------------------------------------------------------\n");
khan_data = (khan_state*)calloc(sizeof(struct khan_state), 1);
if (khan_data == NULL) {
//log_msg("Could not allocate memory to khan_data!..Aborting..!\n");
abort();
}
if(initializing_khan(argv[1])<0) {
//log_msg("Could not initialize khan..Aborting..!\n");
return -1;
}
rename_times.open(rename_times_file_name.c_str(), ofstream::out);
rename_times.precision(15);
start_times.open(start_times_file_name.c_str(), ofstream::out);
start_times.precision(15);
//log_msg("initialized....");
retval=fuse_main(args.argc,args.argv, &khan_ops, khan_data);
//log_msg("Done with fuse_main...\n");
return retval;
}
int main(int argc, char **argv)
{
struct sigaction act;
memset (&act, '\0', sizeof(act));
act.sa_sigaction = &cleanup_handler;
act.sa_flags = 0;
measurements_init();
EVclient test_client;
EVclient_sinks sink_capabilities;
EVclient_sources source_capabilities;
(void)argc; (void)argv;
cm = CManager_create();
CMlisten(cm);
char master_address[200];
dfg_get_master_contact_func(master_address,"master.info");
char source_node[300] = "src_";
strcat(source_node, argv[1]);
stor_source_handle = EVcreate_submit_handle(cm, -1, simple_format_list);
source_capabilities = EVclient_register_source(source_node, stor_source_handle);
sink_capabilities = EVclient_register_sink_handler(cm, "sink_b", simple_format_list,
(EVSimpleHandlerFunc) simple_handler, NULL);
/* We're node "a" in the DFG */
test_client = EVclient_assoc(cm, argv[1], master_address, source_capabilities, sink_capabilities);
if (EVclient_ready_wait(test_client) != 1) {
/* dfg initialization failed! */
exit(1);
}
/**************************/
if (sigaction(SIGINT, &act, NULL) < 0) {
perror ("sigaction");
return 1;
}
//signal(SIGUSR1, create_graphs);
Py_SetProgramName(argv[0]); /* optional but recommended */
Py_Initialize();
PyObject *sys = PyImport_ImportModule("sys");
PyObject *path = PyObject_GetAttrString(sys, "path");
char cwd[1024];
std::string py = "/PyScripts";
std::string pyscripts_path = strdup(getcwd(cwd, sizeof(cwd))) + py;
PyList_Append(path, PyString_FromString(pyscripts_path.c_str()));
PySys_SetObject("path", path);
std::string store_filename="stores.txt"; /* Default */
int port = 6379;
int opt;
std::string host = "localhost";
while ((opt = getopt (argc, argv, "p:s:h:")) != -1)
{
switch (opt)
{
case 'p':
port = atoi(optarg);
break;
case 's':
store_filename = optarg;
break;
case 'h':
host = optarg;
break;
}
}
/* Set signal handler */
//signal(SIGTERM, cleanup_handler);
//signal(SIGKILL, cleanup_handler);
//signal(SIGSEGV, cleanup_handler);
FILE* stores = fopen(store_filename.c_str(), "r");
char buffer[100];
char buffer2[100];
fscanf(stores, "%s\n", buffer);
this_server_id = buffer;
while(fscanf(stores, "%s %s\n", buffer, buffer2)!=EOF) {
servers.push_back(buffer);
server_ids.push_back(buffer2);
if(this_server_id == buffer2) {
this_server = buffer;
}
}
fclose(stores);
arg_struct khan_args;
khan_args.mnt_dir = argv[1];
khan_args.servers = servers;
khan_args.server_ids = server_ids;
khan_args.port = port;
khan_args.host = host;
initializing_khan((void*)&khan_args);
log_info("Initialized Khan");
CMrun_network(cm);
return 0;
}
