bool send_message(int socket, StingerBatch & batch) {
google::protobuf::uint64 batch_length = batch.ByteSize();
int64_t prefix_length = sizeof(batch_length);
int64_t buffer_length = prefix_length + batch_length;
google::protobuf::uint8 * buffer = new google::protobuf::uint8 [buffer_length];
google::protobuf::io::ArrayOutputStream array_output(buffer, buffer_length);
google::protobuf::io::CodedOutputStream coded_output(&array_output);
coded_output.WriteLittleEndian64(batch_length);
batch.SerializeToCodedStream(&coded_output);
int sent_bytes = write(socket, buffer, buffer_length);
delete [] buffer;
if (sent_bytes != buffer_length) {
return false;
}
return true;
}
int main(int argc, char *argv[])
{
/* global options */
int port = 10102;
struct hostent * server = NULL;
int opt = 0;
uint64_t batch_time = 0;
uint64_t batch_size = 0;
while(-1 != (opt = getopt(argc, argv, "p:a:s:t:"))) {
switch(opt) {
case 'p': {
port = atoi(optarg);
} break;
case 'a': {
server = gethostbyname(optarg);
if(NULL == server) {
LOG_E_A ("ERROR: server %s could not be resolved.", optarg);
exit(-1);
}
} break;
case '?':
case 'h': {
printf("Usage: %s [-p port] [-a server_addr] [[-t batch_delay (seconds)]|[-s batch_size]]\n", argv[0]);
printf("Defaults:\n\tport: %d\n\tserver: localhost\n\tdelay: %f seconds\nbatch_size: %d", port, NETFLOW_BATCH_TIME, NETFLOW_BATCH_SIZE);
exit(0);
} break;
// Time between netflow batches
case 't': {
batch_time = atoi(optarg);
} break;
// Size of netflow batches
case 's': {
batch_size = atoi(optarg);
} break;
}
}
LOG_D_A ("Running with: port: %d", port);
/* connect to localhost if server is unspecified */
if(NULL == server) {
server = gethostbyname("localhost");
if(NULL == server) {
LOG_E_A ("ERROR: server %s could not be resolved.", "localhost");
exit (-1);
}
}
if(batch_time == 0){
batch_time = NETFLOW_BATCH_TIME;
}
if(batch_size == 0){
batch_size = NETFLOW_BATCH_SIZE;
}
LOG_D_A ("Sending batches every %d seconds", batch_time);
/* start the connection */
int sock_handle = connect_to_batch_server (server, port);
/* actually generate and send the batches */
int64_t batch_num = 0;
int line_count;
char *result;
char input_line[MAX_LINE];
std::string line;
StingerBatch batch;
batch.set_make_undirected(true);
batch.set_type(STRINGS_ONLY);
batch.set_keep_alive(true);
std::vector<EdgeInsertion> batch_buffer;
int64_t insertIdx =0;
int64_t edgeIdx =0;
double global_start = dpc_tic();
double start = dpc_tic();
double timeInSeconds = dpc_toc(start);
while((result = fgets(input_line, MAX_LINE, stdin )) != NULL) {
NetflowParse p;
if(p.isHeader(input_line)) {
continue;
}
p.parseLine(input_line);
/*
// DEBUG Ensure unique entry
std::stringstream bnum;
bnum << batch_num;
if(batch_num % 3 == 0) {
p.src += bnum.str();
p.dest += bnum.str();
}
// END Unique
*/
EdgeInsertion * insertion = batch.add_insertions();
insertion->set_source_str(p.src);
insertion->set_destination_str(p.dest);
insertion->set_type_str(p.protocol);
insertion->set_weight(p.bytes);
insertion->set_time(p.time);
if (ferror(stdin)) {
perror("Error reading stdin.");
}
// Assume batches are separated by number of insertions if batch_time is 0 or negative
if(batch_time <= 0) {
if (batch_num == 0 || batch_num % batch_size == 0) {
int batch_size = batch.insertions_size() + batch.deletions_size();
LOG_D ("Sending message");
LOG_D_A ("%ld: %d actions. EX: %s:<%s, %s> (w: %d) at time %ld", batch_num, batch_size, p.protocol.c_str(), p.src.c_str(), p.dest.c_str(), p.bytes, p.time);
send_message(sock_handle, batch);
batch.clear_insertions();
batch_num++;
start = dpc_tic();
} else {
batch_buffer.push_back(*insertion);
}
}
// Process new batch of insertions after batch_time seconds
else {
timeInSeconds = dpc_toc(start);
if(timeInSeconds > batch_time){
int batch_size = batch.insertions_size() + batch.deletions_size();
// LOG_D_A ("%f Seconds, %ld Batches", timeInSeconds, batch_num);
LOG_D_A ("%ld: %d actions. EX: %s:<%s, %s> (w: %d) at time %ld after %f seconds", batch_num, batch_size, p.protocol.c_str(), p.src.c_str(), p.dest.c_str(), p.bytes, p.time, dpc_toc(start));
send_message(sock_handle, batch);
batch.clear_insertions();
batch_buffer.clear();
start = dpc_tic();
batch_num++;
} else {
batch_buffer.push_back(*insertion);
}
}
insertIdx++;
}
LOG_D_A("Sent %ld batches in %f seconds. Constructing last batch from buffer ", batch_num, dpc_toc(global_start));
batch.clear_insertions();
for (std::vector<EdgeInsertion>::iterator it = batch_buffer.begin() ; it != batch_buffer.end(); ++it) {
EdgeInsertion * insertion = batch.add_insertions();
insertion->set_source_str(it->source_str());
insertion->set_destination_str(it->destination_str());
insertion->set_type_str(it->type_str());
insertion->set_weight(it->weight());
insertion->set_time(it->time());
// LOG_I_A ("%s:<%s, %s> (w: %ld) at time %ld", insertion->type_str().c_str(), insertion->source_str().c_str(), insertion->destination_str().c_str(), insertion->weight(), insertion->time());
}
send_message(sock_handle, batch);
LOG_D_A("Sent %ld messages from buffer. Total time: %f",batch_buffer.size(), dpc_toc(global_start));
return 0;
}
int
main(int argc, char *argv[])
{
/* global options */
int port = 10102;
int batch_size = 100000;
int num_batches = -1;
int nv = 1024;
int is_int = 0;
struct hostent * server = NULL;
int opt = 0;
while(-1 != (opt = getopt(argc, argv, "p:a:x:y:n:i"))) {
switch(opt) {
case 'p': {
port = atoi(optarg);
} break;
case 'x': {
batch_size = atol(optarg);
} break;
case 'y': {
num_batches = atol(optarg);
} break;
case 'a': {
server = gethostbyname(optarg);
if(NULL == server) {
E_A("ERROR: server %s could not be resolved.", optarg);
exit(-1);
}
} break;
case 'i': {
is_int = 1;
} break;
case 'n': {
nv = atol(optarg);
} break;
case '?':
case 'h': {
printf("Usage: %s [-p port] [-a server_addr] [-n num_vertices] [-x batch_size] [-y num_batches] [-i]\n", argv[0]);
printf("Defaults:\n\tport: %d\n\tserver: localhost\n\tnum_vertices: %d\n-i forces the use of integers in place of strings\n", port, nv);
exit(0);
} break;
}
}
V_A("Running with: port: %d\n", port);
/* connect to localhost if server is unspecified */
if(NULL == server) {
server = gethostbyname("localhost");
if(NULL == server) {
E_A("ERROR: server %s could not be resolved.", "localhost");
exit(-1);
}
}
if(!(nv > 0)) {
printf("ERROR: incorrect number of vertices");
exit(-1);
}
/* start the connection */
int sock_handle = connect_to_batch_server (server, port);
/* actually generate and send the batches */
int64_t line = 0;
int64_t batch_num = 0;
srand (time(NULL));
int64_t scale = 0;
int64_t tmp_nv = nv;
while (tmp_nv >>= 1) ++scale;
double a = 0.55;
double b = 0.15;
double c = 0.15;
double d = 0.25;
dxor128_env_t env;
dxor128_seed(&env, 0);
while(1) {
StingerBatch batch;
batch.set_make_undirected(true);
batch.set_type(is_int ? NUMBERS_ONLY : STRINGS_ONLY);
batch.set_keep_alive(true);
std::string src, dest;
// IF YOU MAKE THIS LOOP PARALLEL,
// MOVE THE SRC/DEST STRINGS ABOVE
for(int e = 0; e < batch_size; e++) {
line++;
int64_t u, v;
rmat_edge (&u, &v, scale, a, b, c, d, &env);
if(u == v) {
e--;
line--;
continue;
}
/* is insert? */
EdgeInsertion * insertion = batch.add_insertions();
if(is_int) {
insertion->set_source(u);
insertion->set_destination(v);
} else {
insertion->set_source_str(build_name(src, u));
insertion->set_destination_str(build_name(dest, v));
}
insertion->set_weight(1);
insertion->set_time(line);
}
V("Sending messages.");
send_message(sock_handle, batch);
sleep(2);
batch_num++;
if((batch_num >= num_batches) && (num_batches != -1)) {
break;
}
}
StingerBatch batch;
batch.set_make_undirected(true);
batch.set_type(is_int ? NUMBERS_ONLY : STRINGS_ONLY);
batch.set_keep_alive(false);
send_message(sock_handle, batch);
return 0;
}
int
main(int argc, char *argv[])
{
/* global options */
int port = 10102;
int batch_size = 1000;
double timeout = 0;
struct hostent * server = NULL;
char * filename = NULL;
int use_directed = 0;
int opt = 0;
while(-1 != (opt = getopt(argc, argv, "p:a:x:t:d"))) {
switch(opt) {
case 'p': {
port = atoi(optarg);
} break;
case 'x': {
batch_size = atol(optarg);
LOG_I_A("Batch size changed to %d", batch_size);
} break;
case 'a': {
server = gethostbyname(optarg);
if(NULL == server) {
LOG_E_A("ERROR: server %s could not be resolved.", optarg);
exit(-1);
}
} break;
case 'd': {
use_directed = 1;
} break;
case 't': {
timeout = atof(optarg);
} break;
case '?':
case 'h': {
printf("Usage: %s [-p port] [-a server_addr] [-t timeout] [-x batch_size] filename\n", argv[0]);
printf("Defaults:\n\tport: %d\n\tserver: localhost\n\ttimeout:%lf\n\tbatch_size: %d", port, timeout, batch_size);
exit(0);
} break;
}
}
if (optind < argc && 0 != strcmp (argv[optind], "-")) {
filename = argv[optind];
} else {
LOG_E("No filename given.");
return -1;
}
LOG_V_A("Running with: port: %d\n", port);
/* connect to localhost if server is unspecified */
if(NULL == server) {
server = gethostbyname("localhost");
if(NULL == server) {
LOG_E_A("ERROR: server %s could not be resolved.", "localhost");
exit(-1);
}
}
/* start the connection */
int sock_handle = connect_to_batch_server (server, port);
EdgeCollectionSet edge_finder;
FILE * fp = fopen(filename, "r");
char *line = NULL;
size_t linecap = 0;
ssize_t linelen;
rapidjson::Document document;
while ((linelen = getdelim(&line, &linecap, '\r', fp)) > 0) {
document.ParseInsitu<0>(line);
if(document.IsObject())
edge_finder.learn(document);
}
LOG_V("Printing learn");
edge_finder.print();
StingerBatch batch;
if(use_directed) {
batch.set_make_undirected(false);
} else {
batch.set_make_undirected(true);
}
batch.set_type(MIXED);
batch.set_keep_alive(true);
tic();
double timesince = 0;
while ((linelen = getdelim(&line, &linecap, '\r', stdin)) > 0) {
document.Parse<0>(line);
if(document.IsObject()) {
if(edge_finder.apply(batch, document, batch.metadata_size())) {
batch.add_metadata(line);
}
}
timesince += toc();
int64_t total_actions = batch.insertions_size() + batch.deletions_size();
if(total_actions >= batch_size || (timeout > 0 && timesince >= timeout)) {
LOG_I_A("Sending a batch of %ld actions", total_actions);
send_message(sock_handle, batch);
timesince = 0;
batch.Clear();
if(use_directed) {
batch.set_make_undirected(false);
} else {
batch.set_make_undirected(true);
}
batch.set_type(MIXED);
batch.set_keep_alive(true);
sleep(5);
}
}
int64_t total_actions = batch.insertions_size() + batch.deletions_size();
if(total_actions) {
LOG_I_A("Sending a batch of %ld actions", total_actions);
send_message(sock_handle, batch);
}
return 0;
}
int
main(int argc, char *argv[])
{
/* global options */
int port = 10102;
int batch_size = 1000;
double timeout = 0;
char * hostname = NULL;
char * filename = NULL;
int use_directed = 0;
int opt = 0;
while(-1 != (opt = getopt(argc, argv, "p:a:x:t:d"))) {
switch(opt) {
case 'p': {
port = atoi(optarg);
} break;
case 'x': {
batch_size = atol(optarg);
LOG_I_A("Batch size changed to %d", batch_size);
} break;
case 'a': {
hostname = optarg;
} break;
case 'd': {
use_directed = 1;
} break;
case 't': {
timeout = atof(optarg);
} break;
case '?':
case 'h': {
printf("Usage: %s [-p port] [-a server_addr] [-t timeout] [-x batch_size] filename\n", argv[0]);
printf("Defaults:\n\tport: %d\n\tserver: localhost\n\ttimeout:%lf\n\tbatch_size: %d", port, timeout, batch_size);
exit(0);
} break;
}
}
if (optind < argc && 0 != strcmp (argv[optind], "-")) {
filename = argv[optind];
} else {
LOG_E("No filename given.");
return -1;
}
LOG_V_A("Running with: port: %d\n", port);
/* connect to localhost if server is unspecified */
if(NULL == hostname) {
hostname = "localhost";
}
/* start the connection */
int sock_handle = connect_to_server (hostname, port);
if (sock_handle == -1) exit(-1);
EdgeCollectionSet edge_finder;
FILE * fp = fopen(filename, "r");
char * buf = NULL, ** fields = NULL;
uint64_t bufSize = 0, * lengths = NULL, fieldsSize = 0, count = 0;
while (!feof(fp)) {
readCSVLineDynamic(',', fp, &buf, &bufSize, &fields, &lengths, &fieldsSize, &count);
if (count <= 1)
continue;
edge_finder.learn(fields, (int64_t *)lengths, count);
}
LOG_V("Printing learn");
edge_finder.print();
StingerBatch batch;
if(use_directed) {
batch.set_make_undirected(false);
} else {
batch.set_make_undirected(true);
}
batch.set_type(MIXED);
batch.set_keep_alive(true);
tic();
double timesince = 0;
while (!feof(stdin)) {
int64_t count_read = readCSVLineDynamic(',', stdin, &buf, &bufSize, &fields, &lengths, &fieldsSize, &count);
if (count > 1) {
if(edge_finder.apply(batch, fields, (int64_t *)lengths, count, batch.metadata_size())) {
batch.add_metadata(buf, count_read);
}
}
timesince += toc();
int64_t total_actions = batch.insertions_size() + batch.deletions_size();
if(total_actions >= batch_size || (timeout > 0 && timesince >= timeout)) {
LOG_I_A("Sending a batch of %ld actions", total_actions);
send_message(sock_handle, batch);
timesince = 0;
batch.Clear();
if(use_directed) {
batch.set_make_undirected(false);
} else {
batch.set_make_undirected(true);
}
batch.set_type(MIXED);
batch.set_keep_alive(true);
}
}
int64_t total_actions = batch.insertions_size() + batch.deletions_size();
if(total_actions) {
LOG_I_A("Sending a batch of %ld actions", total_actions);
send_message(sock_handle, batch);
}
free(buf); free(fields); free(lengths);
return 0;
}
int
main(int argc, char *argv[])
{
/* global options */
int port = 10102;
struct hostent * server = NULL;
int opt = 0;
while(-1 != (opt = getopt(argc, argv, "p:a:"))) {
switch(opt) {
case 'p': {
port = atoi(optarg);
} break;
case 'a': {
server = gethostbyname(optarg);
if(NULL == server) {
LOG_E_A ("ERROR: server %s could not be resolved.", optarg);
exit(-1);
}
} break;
case '?':
case 'h': {
printf("Usage: %s [-p port] [-a server_addr]\n", argv[0]);
printf("Defaults:\n\tport: %d\n\tserver: localhost\n", port);
exit(0);
} break;
}
}
LOG_D_A ("Running with: port: %d", port);
/* connect to localhost if server is unspecified */
if(NULL == server) {
server = gethostbyname("localhost");
if(NULL == server) {
LOG_E_A ("ERROR: server %s could not be resolved.", "localhost");
exit (-1);
}
}
/* start the connection */
int sock_handle = connect_to_batch_server (server, port);
/* actually generate and send the batches */
int64_t batch_num = 0;
int64_t time = 0;
while (1)
{
StingerBatch batch;
batch.set_make_undirected(true);
batch.set_type(STRINGS_ONLY);
batch.set_keep_alive(true);
std::string src, dest;
std::cout << "\nEnter the source vertex: ";
std::getline (std::cin, src);
std::cout << "Enter the destination vertex: ";
std::getline (std::cin, dest);
time = get_current_timestamp();
EdgeInsertion * insertion = batch.add_insertions();
insertion->set_source_str(src);
insertion->set_destination_str(dest);
insertion->set_weight(1);
insertion->set_time(time);
LOG_I_A ("%ld: <%s, %s> at time %ld", batch_num, src.c_str(), dest.c_str(), time);
LOG_D ("Sending message");
send_message(sock_handle, batch);
batch_num++;
}
StingerBatch batch;
batch.set_make_undirected(true);
batch.set_type(STRINGS_ONLY);
batch.set_keep_alive(false);
send_message(sock_handle, batch);
return 0;
}
int
main(int argc, char *argv[])
{
/* global options */
int port = 10102;
int batch_size = 1000;
double timeout = 0;
struct hostent * server = NULL;
char * filename = NULL;
int opt = 0;
while(-1 != (opt = getopt(argc, argv, "p:a:xt:"))) {
switch(opt) {
case 'p': {
port = atoi(optarg);
} break;
case 'x': {
batch_size = atol(optarg);
} break;
case 'a': {
server = gethostbyname(optarg);
if(NULL == server) {
E_A("ERROR: server %s could not be resolved.", optarg);
exit(-1);
}
} break;
case 't': {
timeout = atof(optarg);
} break;
case '?':
case 'h': {
printf("Usage: %s [-p port] [-a server_addr] [-t timeout] [-x batch_size] filename\n", argv[0]);
printf("Defaults:\n\tport: %d\n\tserver: localhost\n\ttimeout:%lf\n\tbatch_size: %d", port, timeout, batch_size);
exit(0);
} break;
}
}
if (optind < argc && 0 != strcmp (argv[optind], "-"))
filename = argv[optind];
V_A("Running with: port: %d\n", port);
/* connect to localhost if server is unspecified */
if(NULL == server) {
server = gethostbyname("localhost");
if(NULL == server) {
E_A("ERROR: server %s could not be resolved.", "localhost");
exit(-1);
}
}
/* start the connection */
int sock_handle = connect_to_batch_server (server, port);
EdgeCollectionSet edge_finder;
FILE * fp = fopen(filename, "r");
char * buf = NULL, ** fields = NULL;
uint64_t bufSize = 0, * lengths = NULL, fieldsSize = 0, count = 0;
while (!feof(fp)) {
readCSVLineDynamic(',', fp, &buf, &bufSize, &fields, &lengths, &fieldsSize, &count);
if (count <= 1)
continue;
edge_finder.learn(fields, (int64_t *)lengths, count);
}
LOG_V("Printing learn");
edge_finder.print();
StingerBatch batch;
batch.set_make_undirected(true);
batch.set_type(MIXED);
batch.set_keep_alive(true);
tic();
double timesince = 0;
while (!feof(stdin)) {
readCSVLineDynamic(',', stdin, &buf, &bufSize, &fields, &lengths, &fieldsSize, &count);
if (count > 1)
edge_finder.apply(batch, fields, (int64_t *)lengths, count);
timesince += toc();
int64_t total_actions = batch.insertions_size() + batch.deletions_size();
if(total_actions >= batch_size || (timeout > 0 && timesince >= timeout)) {
LOG_I_A("Sending a batch of %ld actions", total_actions);
send_message(sock_handle, batch);
timesince = 0;
batch.Clear();
batch.set_make_undirected(true);
batch.set_type(MIXED);
batch.set_keep_alive(true);
}
}
int64_t total_actions = batch.insertions_size() + batch.deletions_size();
if(total_actions) {
LOG_I_A("Sending a batch of %ld actions", total_actions);
send_message(sock_handle, batch);
}
free(buf); free(fields); free(lengths);
return 0;
}
