JniNdbEventStreamingImp::JniNdbEventStreamingImp(JNIEnv *env,jboolean jIsLeader, jstring jConnectionString,
jstring jDatabaseName){
LOG_INFO("start table tailers");
mPollMaxTimeToWait=100;
const char *connectionString=env->GetStringUTFChars(jConnectionString, 0);
mClusterConnection = connect_to_cluster(connectionString);
const char *databaseName=env->GetStringUTFChars(jDatabaseName, 0);
JavaVM* jvm;
env->GetJavaVM(&jvm);
isLeader=(bool) jIsLeader;
if(isLeader){
Ndb* rmnode_tailer_connection = create_ndb_connection(databaseName);
rmNodeTailer = new RMNodeTableTailer(rmnode_tailer_connection, mPollMaxTimeToWait,jvm);
rmNodeTailer->start();
Ndb* pendingEvent_tailer_connection = create_ndb_connection(databaseName);
pendingEventTailer = new PendingEventTableTailer(pendingEvent_tailer_connection, mPollMaxTimeToWait,jvm);
pendingEventTailer->start();
Ndb* resource_tailer_connection = create_ndb_connection(databaseName);
resourceTailer = new ResourceTableTailer(resource_tailer_connection, mPollMaxTimeToWait,jvm);
resourceTailer->start();
Ndb* updatedContainerInfo_tailer_connection = create_ndb_connection(databaseName);
updatedContainerInfoTailer = new UpdatedContainerInfoTableTailer(updatedContainerInfo_tailer_connection, mPollMaxTimeToWait,jvm);
updatedContainerInfoTailer->start();
Ndb* containerStatus_tailer_connection = create_ndb_connection(databaseName);
containerStatusTailer = new ContainerStatusTableTailer(containerStatus_tailer_connection, mPollMaxTimeToWait,jvm);
containerStatusTailer->start();
}else{
Ndb* containerIdToClean_tailer_connection = create_ndb_connection(databaseName);
containerIdToCleanTailer = new ContainerIdToCleanTableTailer(containerIdToClean_tailer_connection, mPollMaxTimeToWait,jvm);
containerIdToCleanTailer->start();
Ndb* containerToSignal_tailer_connection = create_ndb_connection(databaseName);
containerToSignalTailer = new ContainerToSignalTableTailer(containerToSignal_tailer_connection, mPollMaxTimeToWait,jvm);
containerToSignalTailer->start();
Ndb* containerToDecrease_tailer_connection = create_ndb_connection(databaseName);
containerToDecreaseTailer = new ContainerToDecreaseTableTailer(containerToDecrease_tailer_connection, mPollMaxTimeToWait,jvm);
containerToDecreaseTailer->start();
Ndb* nextHeartBeat_tailer_connection = create_ndb_connection(databaseName);
nextHeartBeatTailer = new NextHeartBeatTableTailer(nextHeartBeat_tailer_connection, mPollMaxTimeToWait,jvm);
nextHeartBeatTailer->start();
Ndb* rmNodeApplications_tailer_connection = create_ndb_connection(databaseName);
rmNodeApplicationsTailer = new RmNodeApplicationsTableTailer(rmNodeApplications_tailer_connection, mPollMaxTimeToWait,jvm);
rmNodeApplicationsTailer->start();
}
}
void Notifier::start() {
LOG_INFO("ePipe starting...");
ptime t1 = getCurrentTime();
RecoveryIndeces ri;
if(mRecovery){
Ndb* recovery_conn = create_ndb_connection(mMetaDatabaseName);
ri = Recovery::getRecoveryIndeces(recovery_conn);
}
mFsMutationsDataReader->start();
if(mMetadataType == SchemaBased || mMetadataType == Both){
mMetadataReader->start();
}
if(mMetadataType == Schemaless || mMetadataType == Both){
mSchemalessMetadataReader->start();
}
mElasticSearch->start();
//TODO: ePipe cannot capture project/dataset/metadata deletion events
// when it is down.
// new projects/datasets/metadata/files that
// were created while ePipe was starting are not captured as well.
//
if (mHopsworksEnabled) {
mProjectTableTailer->start(ri.mProjectIndex);
mDatasetTableTailer->start(ri.mDatasetIndex);
}
mFsMutationsBatcher->start();
mFsMutationsTableTailer->start(ri.mMutationsIndex);
if (mMetadataType == SchemaBased || mMetadataType == Both) {
mMetadataBatcher->start();
mMetadataTableTailer->start(ri.mMetadataIndex);
}
if (mMetadataType == Schemaless || mMetadataType == Both) {
mSchemalessMetadataBatcher->start();
mSchemalessMetadataTailer->start(ri.mSchemalessMetadataIndex);
}
ptime t2 = getCurrentTime();
LOG_INFO("ePipe started in " << getTimeDiffInMilliseconds(t1, t2) << " msec");
mFsMutationsBatcher->waitToFinish();
mMetadataBatcher->waitToFinish();
mElasticSearch->waitToFinish();
}
void Notifier::setup() {
mPDICache = new ProjectDatasetINodeCache(mLRUCap);
mElasticSearch = new ElasticSearch(mElasticAddr, mElasticIndex, mElastticProjectType,
mElasticDatasetType, mElasticInodeType, mElasticIssueTime, mElasticBatchsize, mStats);
Ndb* mutations_tailer_connection = create_ndb_connection(mDatabaseName);
mFsMutationsTableTailer = new FsMutationsTableTailer(mutations_tailer_connection, mPollMaxTimeToWait, mPDICache);
MConn* mutations_connections = new MConn[mMutationsTU.mNumReaders];
for(int i=0; i< mMutationsTU.mNumReaders; i++){
mutations_connections[i].inodeConnection = create_ndb_connection(mDatabaseName);
mutations_connections[i].metadataConnection = create_ndb_connection(mMetaDatabaseName);
}
mFsMutationsDataReader = new FsMutationsDataReader(mutations_connections, mMutationsTU.mNumReaders,
mHopsworksEnabled, mElasticSearch, mPDICache, mLRUCap);
mFsMutationsBatcher = new FsMutationsBatcher(mFsMutationsTableTailer, mFsMutationsDataReader,
mMutationsTU.mWaitTime, mMutationsTU.mBatchSize);
if (mMetadataType == SchemaBased || mMetadataType == Both) {
Ndb* metadata_tailer_connection = create_ndb_connection(mMetaDatabaseName);
mMetadataTableTailer = new MetadataTableTailer(metadata_tailer_connection, mPollMaxTimeToWait);
MConn* metadata_connections = new MConn[mMetadataTU.mNumReaders];
for (int i = 0; i < mMetadataTU.mNumReaders; i++) {
metadata_connections[i].inodeConnection = create_ndb_connection(mDatabaseName);
metadata_connections[i].metadataConnection = create_ndb_connection(mMetaDatabaseName);
}
mMetadataReader = new MetadataReader(metadata_connections, mMetadataTU.mNumReaders,
mHopsworksEnabled, mElasticSearch, mPDICache, mLRUCap);
mMetadataBatcher = new MetadataBatcher(mMetadataTableTailer, mMetadataReader,
mMetadataTU.mWaitTime, mMetadataTU.mBatchSize);
}
if (mMetadataType == Schemaless || mMetadataType == Both) {
Ndb* s_metadata_tailer_connection = create_ndb_connection(mMetaDatabaseName);
mSchemalessMetadataTailer = new SchemalessMetadataTailer(s_metadata_tailer_connection, mPollMaxTimeToWait);
MConn* s_metadata_connections = new MConn[mSchemalessTU.mNumReaders];
for (int i = 0; i < mSchemalessTU.mNumReaders; i++) {
s_metadata_connections[i].inodeConnection = create_ndb_connection(mDatabaseName);
s_metadata_connections[i].metadataConnection = create_ndb_connection(mMetaDatabaseName);
}
mSchemalessMetadataReader = new SchemalessMetadataReader(s_metadata_connections, mSchemalessTU.mNumReaders,
mHopsworksEnabled, mElasticSearch, mPDICache);
mSchemalessMetadataBatcher = new SchemalessMetadataBatcher(mSchemalessMetadataTailer,
mSchemalessMetadataReader, mSchemalessTU.mWaitTime, mSchemalessTU.mBatchSize);
}
if (mHopsworksEnabled) {
Ndb* project_tailer_connection = create_ndb_connection(mMetaDatabaseName);
mProjectTableTailer = new ProjectTableTailer(project_tailer_connection, mPollMaxTimeToWait,
mElasticSearch, mPDICache);
Ndb* dataset_tailer_connection = create_ndb_connection(mMetaDatabaseName);
mDatasetTableTailer = new DatasetTableTailer(dataset_tailer_connection, mPollMaxTimeToWait,
mElasticSearch, mPDICache);
}
}
