ret_t
cherokee_logger_writer_get_id (cherokee_config_node_t *config,
cherokee_buffer_t *id)
{
ret_t ret;
cherokee_buffer_t *tmp;
cherokee_logger_writer_types_t type;
ret = config_read_type (config, &type);
if (ret != ret_ok) {
return ret;
}
switch (type) {
case cherokee_logger_writer_syslog:
cherokee_buffer_add_str (id, "syslog");
break;
case cherokee_logger_writer_stderr:
cherokee_buffer_add_str (id, "stderr");
break;
case cherokee_logger_writer_file:
ret = cherokee_config_node_read (config, "filename", &tmp);
if (ret == ret_ok) {
cherokee_buffer_add_buffer (id, tmp);
} else {
cherokee_buffer_add_str (id, "unknown");
}
break;
case cherokee_logger_writer_pipe:
ret = cherokee_config_node_read (config, "command", &tmp);
if (ret == ret_ok) {
cherokee_buffer_add_buffer (id, tmp);
} else {
cherokee_buffer_add_str (id, "unknown");
}
break;
default:
SHOULDNT_HAPPEN;
return ret_error;
}
return ret_ok;
}
int main(int argc, char *argv[])
{
/*
* We want the dfg creator file to be able to read in a list of nodes, a list of stones per node and links between stones.
* */
if(argc != 2)
usage();
else
{
std::vector<std::string> type_names;
std::string config_file_name = argv[1];
ConfigParser_t cfg;
if(cfg.readFile(config_file_name))
{
printf("Error: Cannot open config file %s", config_file_name.c_str());
return 1;
}
/*Read the file information into the appropriate data structures for later use*/
std::vector<std::string> stone_sections = cfg.getSections();
std::vector<std::string> temp_node_name_list;
for(std::vector<std::string>::iterator I = stone_sections.begin(), E = stone_sections.end(); I != E; ++I)
{
stone_struct new_stone_struct;
stone_struct pot_python_struct;
/*Get the node name for the stone and in doing so,
bookkeep which node names that we have already seen*/
std::string which_node;
if(!config_read_node(cfg, *I, which_node))
{
fprintf(stderr, "Failure to read node from config\n");
exit(1);
}
unsigned int temp_i = 0;
for(; temp_i < temp_node_name_list.size(); ++temp_i)
{
if(!which_node.compare(temp_node_name_list[temp_i]))
break;
}
if(temp_i == temp_node_name_list.size())
{
temp_node_name_list.push_back(which_node);
++test_dfg.node_count;
}
new_stone_struct.node_name = which_node;
/*Read the stone type into the enum value...*/
if(!config_read_type(cfg, *I, new_stone_struct.stone_type))
{
fprintf(stderr, "Error: reading config type of stone failed\n");
exit(1);
}
/*Store the section name as the stone name
This must be a unique value for every stone*/
if(new_stone_struct.stone_type == PYTHON)
{
if(!setupPythonStones(cfg, *I, new_stone_struct, pot_python_struct))
{
log_err("Error: failed to setup python stones");
exit(1);
}
stone_holder.push_back(new_stone_struct);
stone_holder.push_back(pot_python_struct);
}
else if(new_stone_struct.stone_type == BUCKETROLL)
{
if(!setupBucketStone(cfg, *I, new_stone_struct))
{
log_err("Error: failed to setup bucket stone");
exit(1);
}
stone_holder.push_back(new_stone_struct);
}
else
{
new_stone_struct.stone_name = *I;
/*Construct a unique handler for each source and sink node
For now the nodes will have to read the config file*/
std::string unique_handler_name = new_stone_struct.stone_name + "_" + new_stone_struct.node_name;
new_stone_struct.src_sink_handler_name = unique_handler_name;
/*Read the incoming stones to connect to from the config file*/
if(!config_read_incoming(cfg, *I, new_stone_struct.incoming_stones))
{
log_err("Error reading incoming stones");
exit(1);
}
/*TODO:Read the actual code, probably going to need to
have a seperate code file for the COD code*/
//Push-back here
stone_holder.push_back(new_stone_struct);
/*
printf("Testing: node_name := %s\n", new_stone_struct.node_name.c_str());
printf("Testing: stone_name := %s\n", new_stone_struct.stone_name.c_str());
printf("Testing: handler_name := %s\n", new_stone_struct.src_sink_handler_name.c_str());
printf("Testing: stone_type := %d\n", new_stone_struct.stone_type);
printf("Testing: node_count := %d\n", test_dfg.node_count);
for(unsigned int i = 0; i < new_stone_struct.incoming_stones.size(); ++i)
{
printf("Testing: incoming_stone for %s: %s\n", new_stone_struct.stone_name.c_str(),
new_stone_struct.incoming_stones[i].c_str());
}
printf("Testing: code_type := %d\n", new_stone_struct.code_type);
*/
}
}
// Master node entry stone added to the end of the vector
stone_struct entry_stone_struct;
entry_stone_struct.node_name = "master_node";
entry_stone_struct.stone_name = "entry";
entry_stone_struct.src_sink_handler_name = entry_stone_struct.stone_name + "_" + entry_stone_struct.node_name;
entry_stone_struct.stone_type = SOURCE;
stone_holder.push_back(entry_stone_struct);
++test_dfg.node_count;
if(dfg_init_func())
{
EVmaster_node_join_handler(test_dfg.dfg_master, JoinHandlerFunc);
printf("DFG handler read...\n");
}
/*EVsource * master_list_sources = (EVsource *) malloc(sizeof(EVsource) * temp_node_name_list.size());
for(unsigned int i = 0; i < temp_node_name_list.size(); ++i)
{
//Place the list stones into the list
stone_struct py_master_list_source;
py_master_list_source.node_name = "master_node";
py_master_list_source.stone_name = "to_" + temp_node_name_list[i];
py_master_list_source.src_sink_handler_name = py_master_list_source.stone_name + "_" + py_master_list_source.node_name;
py_master_list_source.stone_type = SOURCE;
stone_holder.push_back(py_master_list_source);
//Register the handler
master_list_sources[i] = EVcreate_submit_handle(test_dfg.cm, -1, python_format_list);
char * perm_ptr = strdup(py_master_list_source.src_sink_handler_name.c_str());
source_capabilities = EVclient_register_source(perm_ptr, master_list_sources[i]);
stone_struct py_master_list_sink;
py_master_list_sink.node_name = temp_node_name_list[i];
py_master_list_sink.stone_name = "python_list";
py_master_list_sink.src_sink_handler_name = py_master_list_sink.stone_name + "_" + py_master_list_sink.node_name;
py_master_list_sink.incoming_stones.push_back(py_master_list_source.stone_name);
py_master_list_sink.stone_type = SINK;
stone_holder.push_back(py_master_list_sink);
}
*/
EVclient_sources source_capabilities;
EVsource source_handle;
EVclient master_client;
source_handle = EVcreate_submit_handle(test_dfg.cm, -1, simple_format_list);
char * temp_char_ptr = strdup(entry_stone_struct.src_sink_handler_name.c_str());
source_capabilities = EVclient_register_source(temp_char_ptr, source_handle);
temp_char_ptr = NULL;
master_client = EVclient_assoc_local(test_dfg.cm, "master_node", test_dfg.dfg_master, source_capabilities, NULL);
if (EVclient_ready_wait(master_client) != 1) {
/* initialization failed! */
log_err("EVclient_ready_wait: FAILED!");
exit(1);
}
/*
python_list test_py_list;
test_py_list.dynamic_size = 4;
test_py_list.ordered_method_list[0] = 0;
test_py_list.ordered_method_list[1] = 1;
test_py_list.ordered_method_list[2] = 2;
test_py_list.ordered_method_list[3] = 3;
for(int i = 0; i < temp_node_name_list.size(); ++i)
{
EVsubmit(master_list_sources[i], &test_py_list, NULL);
printf("Submitted a python list...\n");
}
*/
/* Setting up the options and stuff, before sending anything */
std::string store_filename="stores.txt"; /* Default */
int port = 6379;
int opt;
std::string host = "localhost";
while ((opt = getopt (argc, argv, "p:s:h:")) != -1)
{
switch (opt)
{
case 'p':
port = atoi(optarg);
break;
case 's':
store_filename = optarg;
break;
case 'h':
host = optarg;
break;
}
}
FILE* stores = fopen(store_filename.c_str(), "r");
char buffer[100];
char buffer2[100];
fscanf(stores, "%s\n", buffer);
this_server_id = buffer;
while(fscanf(stores, "%s %s\n", buffer, buffer2)!=EOF) {
servers.push_back(buffer);
server_ids.push_back(buffer2);
if(this_server_id == buffer2) {
this_server = buffer;
}
}
fclose(stores);
/* Initialization of Khan */
arg_struct khan_args;
khan_args.mnt_dir = argv[1];
khan_args.servers = servers;
khan_args.server_ids = server_ids;
khan_args.port = port;
khan_args.host = host;
initializing_khan((void*)&khan_args);
log_info("Initialized Khan");
// Hardcode the im7 type as the only type for now
type_names.push_back("im7");
initialize_attrs_for_data_types(type_names);
/* Here's where we set up and send the data */
simple_rec data;
std::string pattern = servers[0] + "/*";
glob_t files;
int fdin;
struct stat statbuf;
std::set<std::string> experiments;
for(int count = 18; count > 0; count--) {
glob((pattern +".im7").c_str(), 0, NULL, &files);
log_info("Globbing with pattern: %s.im7", pattern.c_str());
for(unsigned j=0; j<files.gl_pathc; j++) {
std::string filepath = files.gl_pathv[j];
log_info("File Path: %s", filepath.c_str());
std::string exp_dir = filepath.substr(0, filepath.size() - 10);
experiments.insert(exp_dir);
data.exp_id = (int)experiments.size();
data.file_path = strdup(filepath.c_str());
if ((fdin = open (filepath.c_str(), O_RDONLY)) < 0)
perror ("can't open %s for reading");
if (fstat (fdin,&statbuf) < 0)
perror ("fstat error");
data.file_buf_len = statbuf.st_size;
log_info("Size: %zu", data.file_buf_len);
if ((data.file_buf = (char*)mmap (0, statbuf.st_size, PROT_READ, MAP_PRIVATE, fdin, 0))
== (caddr_t) -1)
perror ("mmap error for input");
/* Adding in code here to ensure that the event gets setup in Redis correctly before it gets sent out */
//TODO:Delete the bottom two lines if I've compiled successfully without them
//std::string original_fp (data.file_path);
//std::string shared_fn = "/dev/shm/" + original_fp.substr(49, strlen(data.file_path) - 49);
if(!init_database_values(data))
{
log_err("Error in initializing database values for %s", data.file_path);
exit(1);
}
//Set Metadata value to 0
data.meta_compare_py = 0;
EVsubmit(source_handle, &data, NULL);
if (munmap(data.file_buf, statbuf.st_size) == -1) {
perror("Error un-mmapping the file");
}
/*Free Memory*/
close(fdin);
free(data.file_path);
}
pattern += "/*";
}
log_info("Cleanup data structures");
/* Cleanup */
globfree(&files);
log_info("Shutdown evdfg");
CMrun_network(test_dfg.cm);
}
return 0;
}
ret_t
cherokee_logger_writer_configure (cherokee_logger_writer_t *writer, cherokee_config_node_t *config)
{
ret_t ret;
cherokee_buffer_t *tmp = NULL;
/* Check the type
*/
ret = config_read_type (config, &writer->type);
if (ret != ret_ok) {
return ret;
}
/* Extra properties
*/
switch (writer->type) {
case cherokee_logger_writer_file:
ret = cherokee_config_node_read (config, "filename", &tmp);
if (ret != ret_ok) {
LOG_ERROR (CHEROKEE_ERROR_LOGGER_WRITER_READ, "file");
return ret_error;
}
cherokee_buffer_add_buffer (&writer->filename, tmp);
break;
case cherokee_logger_writer_pipe:
ret = cherokee_config_node_read (config, "command", &tmp);
if (ret != ret_ok) {
LOG_ERROR (CHEROKEE_ERROR_LOGGER_WRITER_READ, "exec");
return ret_error;
}
cherokee_buffer_add_buffer (&writer->command, tmp);
break;
default:
break;
}
/* Reside the internal buffer if needed
*/
ret = cherokee_config_node_read (config, "bufsize", &tmp);
if (ret == ret_ok) {
int buf_len;
ret = cherokee_atoi (tmp->buf, &buf_len);
if (ret != ret_ok) {
return ret_error;
}
if (buf_len < LOGGER_MIN_BUFSIZE)
buf_len = LOGGER_MIN_BUFSIZE;
else if (buf_len > LOGGER_MAX_BUFSIZE)
buf_len = LOGGER_MAX_BUFSIZE;
cherokee_buffer_mrproper (&writer->buffer);
cherokee_buffer_init (&writer->buffer);
ret = cherokee_buffer_ensure_size (&writer->buffer, buf_len);
if (ret != ret_ok) {
LOG_ERROR (CHEROKEE_ERROR_LOGGER_WRITER_ALLOC, writer->max_bufsize);
return ret_nomem;
}
writer->max_bufsize = (size_t)buf_len;
}
return ret_ok;
}
