int main(int argc, char* argv[])
{
if (argc < 4) {
std::cout << "Usage: " << argv[0] << "<host> <port> <local/remote>" << std::endl;
exit(1);
}
NetworkOptions options;
options.set_host(argv[1]);
options.set_port(atoi(argv[2]));
options.set_max_packet_size(1024);
if (strcmp(argv[3], "local") == 0) {
options.set_socket_family(PF_UNIX);
options.set_sin_path("/tmp/__echoserver__");
} else {
options.set_socket_family(PF_INET);
}
bool perf = false;
if (argc > 4 && strcmp(argv[4], "perf") == 0)
perf = true;
EventLoopImpl ss;
EchoClient echo_client(&ss, options, perf);
echo_client.Start();
ss.loop();
return 0;
}
static void FOG_CDECL EventLoop_dtor(EventLoop* self)
{
EventLoopImpl* d = self->_d;
if (d)
d->release();
}
int main(int argc, char* argv[]) {
if (argc != 4) {
usage();
exit(1);
}
clustername = argv[1];
sp_string zkhostport = argv[2];
zkroot = argv[3];
// Make sure that zkroot starts with '/'
if (zkroot.substr(0, 1) != "/") {
LOG(ERROR) << "zkroot should start with /" << std::endl;
::exit(1);
}
// remove trailing '/'
if (zkroot[zkroot.size() - 1] == '/') {
zkroot = std::string(zkroot, 0, zkroot.size() - 1);
}
EventLoopImpl ss;
zkclient = new ZKClient(zkhostport, &ss);
zkclient->CreateNode(zkroot, "Heron Cluster " + clustername, false,
[](sp_int32 status) { ZkRootDone(status); });
ss.loop();
}
TEST(PiperTest, test_piper) {
EventLoopImpl* eventLoop = new EventLoopImpl();
std::thread* thread = new std::thread(RunningEventLoop, eventLoop);
Piper* piper = new Piper(eventLoop);
Resource* resource = new Resource();
resource->count = 0;
auto cb = [resource](){ resource->count++; };
piper->ExecuteInEventLoop(cb);
sleep(1);
EXPECT_EQ(resource->count, 1);
piper->ExecuteInEventLoop(cb);
sleep(1);
EXPECT_EQ(resource->count, 2);
eventLoop->loopExit();
// Wait for the thread to terminate
thread->join();
delete piper;
delete resource;
delete eventLoop;
delete thread;
}
int main(int argc, char* argv[])
{
if (argc < 4) {
std::cout << "Usage: " << argv[0] << " <host> <port> <local/remote>" << std::endl;
exit(1);
}
NetworkOptions options;
options.set_host(argv[1]);
options.set_port(atoi(argv[2]));
options.set_max_packet_size(1024);
if (strcmp(argv[3], "local") == 0) {
options.set_socket_family(PF_UNIX);
options.set_sin_path("/tmp/__echoserver__");
} else {
options.set_socket_family(PF_INET);
}
EventLoopImpl ss;
EchoServer echo_server(&ss, options);
if (echo_server.Start() != 0) {
// What the hell happened
std::cout << "Server failed to start\n";
return 1;
}
ss.loop();
return 0;
}
// Test simple data tuples drain
TEST(TupleCache, test_simple_data_drain) {
sp_int32 data_tuples_count = 23354;
EventLoopImpl ss;
sp_uint32 drain_threshold = 1024 * 1024;
heron::stmgr::TupleCache* g = new heron::stmgr::TupleCache(&ss, drain_threshold);
std::map<sp_int32, sp_int32> data_tuples;
data_tuples[1] = data_tuples_count;
std::map<sp_int32, sp_int32> ack_tuples;
std::map<sp_int32, sp_int32> fail_tuples;
Drainer* drainer = new Drainer(data_tuples, ack_tuples, fail_tuples);
g->RegisterDrainer(&Drainer::Drain, drainer);
heron::proto::api::StreamId dummy;
dummy.set_id("stream");
dummy.set_component_name("comp");
for (sp_int32 i = 0; i < data_tuples_count; ++i) {
heron::proto::system::HeronDataTuple tuple;
tuple.set_key(RandUtils::lrand());
g->add_data_tuple(1, dummy, tuple);
}
// 300 milliseconds second
auto cb = [&ss](EventLoopImpl::Status status) { DoneHandler(&ss, status); };
ss.registerTimer(std::move(cb), false, 300000);
ss.loop();
EXPECT_EQ(drainer->Verify(), true);
delete drainer;
delete g;
}
int main(int argc, char* argv[]) {
if (argc != 7) {
std::cout
<< "Usage: " << argv[0]
<< " <host> <master-port> <topology-name> <zk_hostport> <zkroot> <sgmrid:host:port,...>"
<< std::endl;
std::cout << "If zk_hostportlist is empty please say LOCALMODE\n";
::exit(1);
}
std::string myhost = argv[1];
sp_int32 master_port = atoi(argv[2]);
std::string topology_name = argv[3];
std::string zkhostportlist = argv[4];
if (zkhostportlist == "LOCALMODE") {
zkhostportlist = "";
}
std::string topdir = argv[5];
std::vector<std::string> stmgrs = StrUtils::split(argv[6], ",");
EventLoopImpl ss;
NetworkOptions options;
options.set_host(myhost);
options.set_port(master_port);
options.set_max_packet_size(1_MB);
options.set_socket_family(PF_INET);
heron::tmaster::TMasterServer tmaster(&ss, options, topology_name, zkhostportlist, topdir,
stmgrs);
ss.loop();
return 0;
}
static err_t FOG_CDECL EventLoop_quit(EventLoop* self)
{
EventLoopImpl* d = self->_d;
if (d == NULL)
return ERR_RT_INVALID_STATE;
return d->quit();
}
static err_t FOG_CDECL EventLoop_runAllPending(EventLoop* self)
{
EventLoopImpl* d = self->_d;
if (d == NULL)
return ERR_RT_INVALID_STATE;
return d->runAllPending();
}
static err_t FOG_CDECL EventLoop_postTask(EventLoop* self, Task* task, bool nestable, uint32_t delay)
{
EventLoopImpl* d = self->_d;
if (d == NULL)
return ERR_RT_INVALID_STATE;
return d->postTask(task, nestable, delay);
}
static err_t FOG_CDECL EventLoop_removeObserver(EventLoop* self, EventLoopObserver* obj)
{
EventLoopImpl* d = self->_d;
if (d == NULL)
return ERR_RT_INVALID_STATE;
return d->removeObserver(obj);
}
static void FOG_CDECL EventLoop_ctorCopy(EventLoop* self, const EventLoop* other)
{
EventLoopImpl* d = other->_d;
if (d != NULL)
d = d->addRef();
self->_d = d;
}
void start_http_client(sp_uint32 _port, sp_uint64 _requests, sp_uint32 _nkeys) {
port = _port;
ntotal = _requests;
nkeys = _nkeys;
EventLoopImpl ss;
AsyncDNS dns(&ss);
HTTPClient client(&ss, &dns);
SendRequest(&client);
ss.loop();
}
void terminate_server(sp_uint32 port) {
NetworkOptions options;
options.set_host(LOCALHOST);
options.set_port(port);
options.set_max_packet_size(1024 * 1024);
options.set_socket_family(PF_INET);
EventLoopImpl ss;
Terminate ts(&ss, options);
ts.Start();
ss.loop();
}
void start_client(sp_uint32 port, sp_uint64 requests) {
NetworkOptions options;
options.set_host(LOCALHOST);
options.set_port(port);
options.set_max_packet_size(1024 * 1024);
options.set_socket_family(PF_INET);
EventLoopImpl ss;
OrderClient client(&ss, options, requests);
client.Start();
ss.loop();
}
void start_server(sp_uint32 port) {
NetworkOptions options;
options.set_host(LOCALHOST);
options.set_port(port);
options.set_max_packet_size(1024 * 1024);
options.set_socket_family(PF_INET);
EventLoopImpl ss;
server_ = new OrderServer(&ss, options);
if (server_->Start() != 0) GTEST_FAIL();
ss.loop();
}
static err_t FOG_CDECL EventLoop_copy(EventLoop* self, const EventLoop* other)
{
EventLoopImpl* d = other->_d;
if (d)
d = d->addRef();
atomicPtrXchg(&self->_d, d);
if (d)
d->release();
return ERR_OK;
}
void ControlTopology(sp_string topology_id, sp_int32 port, bool activate) {
EventLoopImpl ss;
AsyncDNS dns(&ss);
HTTPClient* client = new HTTPClient(&ss, &dns);
HTTPKeyValuePairs kvs;
kvs.push_back(make_pair("topologyid", topology_id));
sp_string requesturl = activate ? "/activate" : "/deactivate";
OutgoingHTTPRequest* request =
new OutgoingHTTPRequest(LOCALHOST, port, requesturl, BaseHTTPRequest::GET, kvs);
auto cb = [client](IncomingHTTPResponse* response) { ControlTopologyDone(client, response); };
if (client->SendRequest(request, std::move(cb)) != SP_OK) {
FAIL() << "Unable to send the request\n";
}
ss.loop();
}
int main(int argc, char* argv[]) {
if (argc != 12) {
std::cout << "Usage: " << argv[0] << " "
<< "<master-port> <controller-port> <stats-port> "
<< "<topology_name> <topology_id> <zk_hostportlist> "
<< "<topdir> <sgmr1,...> <heron_internals_config_filename> "
<< "<metrics_sinks_filename> <metrics-manager-port>" << std::endl;
std::cout << "If zk_hostportlist is empty please say LOCALMODE\n";
::exit(1);
}
sp_string myhost = IpUtils::getHostName();
sp_int32 master_port = atoi(argv[1]);
sp_int32 controller_port = atoi(argv[2]);
sp_int32 stats_port = atoi(argv[3]);
sp_string topology_name = argv[4];
sp_string topology_id = argv[5];
sp_string zkhostportlist = argv[6];
if (zkhostportlist == "LOCALMODE") {
zkhostportlist = "";
}
sp_string topdir = argv[7];
std::vector<std::string> stmgrs = StrUtils::split(argv[8], ",");
sp_string heron_internals_config_filename = argv[9];
sp_string metrics_sinks_yaml = argv[10];
sp_int32 metrics_manager_port = atoi(argv[11]);
EventLoopImpl ss;
// Read heron internals config from local file
// Create the heron-internals-config-reader to read the heron internals config
heron::config::HeronInternalsConfigReader::Create(&ss, heron_internals_config_filename);
heron::common::Initialize(argv[0], topology_id.c_str());
LOG(INFO) << "Starting tmaster for topology " << topology_name << " with topology id "
<< topology_id << " zkhostport " << zkhostportlist << " zkroot " << topdir
<< " and nstmgrs " << stmgrs.size() << std::endl;
heron::tmaster::TMaster tmaster(zkhostportlist, topology_name, topology_id, topdir, stmgrs,
controller_port, master_port, stats_port, metrics_manager_port,
metrics_sinks_yaml, myhost, &ss);
ss.loop();
return 0;
}
void start_http_server(sp_uint32 _port, sp_uint32 _nkeys, int fd) {
nkeys = _nkeys;
EventLoopImpl ss;
// set host, port and packet size
NetworkOptions options;
options.set_host(LOCALHOST);
options.set_port(_port);
options.set_max_packet_size(BUFSIZ << 4);
// start the server
TestHttpServer http_server(&ss, options);
// use pipe to block clients before server enters event loop
int sent;
write(fd, &sent, sizeof(int));
ss.loop();
}
// Test different stream mix
TEST(TupleCache, test_different_stream_mix) {
sp_int32 data_tuples_count = 23354; // make sure this is even
sp_int32 ack_tuples_count = 3544; // make sure this is even
sp_int32 fail_tuples_count = 6564; // make sure this is even
EventLoopImpl ss;
sp_uint32 drain_threshold = 1024 * 1024;
heron::stmgr::TupleCache* g = new heron::stmgr::TupleCache(&ss, drain_threshold);
std::map<sp_int32, sp_int32> data_tuples;
data_tuples[1] = data_tuples_count / 2;
data_tuples[2] = data_tuples_count / 2;
std::map<sp_int32, sp_int32> ack_tuples;
ack_tuples[1] = ack_tuples_count / 2;
ack_tuples[2] = ack_tuples_count / 2;
std::map<sp_int32, sp_int32> fail_tuples;
fail_tuples[1] = fail_tuples_count / 2;
fail_tuples[2] = fail_tuples_count / 2;
Drainer* drainer = new Drainer(data_tuples, ack_tuples, fail_tuples);
g->RegisterDrainer(&Drainer::Drain, drainer);
heron::proto::api::StreamId stream1;
stream1.set_id("stream1");
stream1.set_component_name("comp1");
heron::proto::api::StreamId stream2;
stream2.set_id("stream2");
stream2.set_component_name("comp2");
sp_int32 max_count = std::max(std::max(data_tuples_count, ack_tuples_count), fail_tuples_count);
for (sp_int32 i = 0; i < max_count; ++i) {
if (i < data_tuples_count) {
heron::proto::system::HeronDataTuple tuple;
tuple.set_key(RandUtils::lrand());
if (i % 2 == 0) {
g->add_data_tuple(1, stream1, tuple);
} else {
g->add_data_tuple(2, stream2, tuple);
}
}
if (i < ack_tuples_count) {
heron::proto::system::AckTuple tuple;
tuple.set_ackedtuple(RandUtils::lrand());
if (i % 2 == 0) {
g->add_ack_tuple(1, tuple);
} else {
g->add_ack_tuple(2, tuple);
}
}
if (i < fail_tuples_count) {
heron::proto::system::AckTuple tuple;
tuple.set_ackedtuple(RandUtils::lrand());
if (i % 2 == 0) {
g->add_fail_tuple(1, tuple);
} else {
g->add_fail_tuple(2, tuple);
}
}
}
// 400 milliseconds second
auto cb = [&ss](EventLoopImpl::Status status) { DoneHandler(&ss, status); };
ss.registerTimer(std::move(cb), false, 300000);
ss.loop();
EXPECT_EQ(drainer->Verify(), true);
delete drainer;
delete g;
}
