JniNdbEventStreamingImp::JniNdbEventStreamingImp(JNIEnv *env,jboolean jIsLeader, jstring jConnectionString,
        jstring jDatabaseName){
  LOG_INFO("start table tailers");
  mPollMaxTimeToWait=100;
  const char *connectionString=env->GetStringUTFChars(jConnectionString, 0);
  mClusterConnection = connect_to_cluster(connectionString);

  const char *databaseName=env->GetStringUTFChars(jDatabaseName, 0);

  JavaVM* jvm;
  env->GetJavaVM(&jvm);

  isLeader=(bool) jIsLeader;
  
  if(isLeader){
    Ndb* rmnode_tailer_connection = create_ndb_connection(databaseName);
    rmNodeTailer = new RMNodeTableTailer(rmnode_tailer_connection, mPollMaxTimeToWait,jvm);
    rmNodeTailer->start();
    
    Ndb* pendingEvent_tailer_connection = create_ndb_connection(databaseName);
    pendingEventTailer = new PendingEventTableTailer(pendingEvent_tailer_connection, mPollMaxTimeToWait,jvm);
    pendingEventTailer->start();
    
    Ndb* resource_tailer_connection = create_ndb_connection(databaseName);
    resourceTailer = new ResourceTableTailer(resource_tailer_connection, mPollMaxTimeToWait,jvm);
    resourceTailer->start();
    
    Ndb* updatedContainerInfo_tailer_connection = create_ndb_connection(databaseName);
    updatedContainerInfoTailer = new UpdatedContainerInfoTableTailer(updatedContainerInfo_tailer_connection, mPollMaxTimeToWait,jvm);
    updatedContainerInfoTailer->start();
    
    Ndb* containerStatus_tailer_connection = create_ndb_connection(databaseName);
    containerStatusTailer = new ContainerStatusTableTailer(containerStatus_tailer_connection, mPollMaxTimeToWait,jvm);
    containerStatusTailer->start();
  }else{
    Ndb* containerIdToClean_tailer_connection = create_ndb_connection(databaseName);
    containerIdToCleanTailer = new ContainerIdToCleanTableTailer(containerIdToClean_tailer_connection, mPollMaxTimeToWait,jvm);
    containerIdToCleanTailer->start();

    Ndb* containerToSignal_tailer_connection = create_ndb_connection(databaseName);
    containerToSignalTailer = new ContainerToSignalTableTailer(containerToSignal_tailer_connection, mPollMaxTimeToWait,jvm);
    containerToSignalTailer->start();

    Ndb* containerToDecrease_tailer_connection = create_ndb_connection(databaseName);
    containerToDecreaseTailer = new ContainerToDecreaseTableTailer(containerToDecrease_tailer_connection, mPollMaxTimeToWait,jvm);
    containerToDecreaseTailer->start();

    Ndb* nextHeartBeat_tailer_connection = create_ndb_connection(databaseName);
    nextHeartBeatTailer = new NextHeartBeatTableTailer(nextHeartBeat_tailer_connection, mPollMaxTimeToWait,jvm);
    nextHeartBeatTailer->start();

    Ndb* rmNodeApplications_tailer_connection = create_ndb_connection(databaseName);
    rmNodeApplicationsTailer = new RmNodeApplicationsTableTailer(rmNodeApplications_tailer_connection, mPollMaxTimeToWait,jvm);
    rmNodeApplicationsTailer->start();
  }
}
Beispiel #2
0
void Notifier::start() {
    LOG_INFO("ePipe starting...");
    ptime t1 = getCurrentTime();
    RecoveryIndeces ri;
    if(mRecovery){
        Ndb* recovery_conn = create_ndb_connection(mMetaDatabaseName);
        ri = Recovery::getRecoveryIndeces(recovery_conn);
    }
    
    mFsMutationsDataReader->start();
    if(mMetadataType == SchemaBased || mMetadataType == Both){
        mMetadataReader->start();
    }
    if(mMetadataType == Schemaless || mMetadataType == Both){
        mSchemalessMetadataReader->start();
    }    
    mElasticSearch->start();
    
    //TODO: ePipe cannot capture project/dataset/metadata deletion events
    //      when it is down.
    //      new projects/datasets/metadata/files that 
    //      were created while ePipe was starting are not captured as well. 
    //      
    if (mHopsworksEnabled) {
        mProjectTableTailer->start(ri.mProjectIndex);
        mDatasetTableTailer->start(ri.mDatasetIndex);
    }
    
    mFsMutationsBatcher->start();
    mFsMutationsTableTailer->start(ri.mMutationsIndex);

    if (mMetadataType == SchemaBased || mMetadataType == Both) {
        mMetadataBatcher->start();
        mMetadataTableTailer->start(ri.mMetadataIndex);
    }

    if (mMetadataType == Schemaless || mMetadataType == Both) {
        mSchemalessMetadataBatcher->start();
        mSchemalessMetadataTailer->start(ri.mSchemalessMetadataIndex);
    }
    
    ptime t2 = getCurrentTime();
    LOG_INFO("ePipe started in " << getTimeDiffInMilliseconds(t1, t2) << " msec");
    mFsMutationsBatcher->waitToFinish();
    mMetadataBatcher->waitToFinish();
    mElasticSearch->waitToFinish();
}
Beispiel #3
0
void Notifier::setup() {
    mPDICache = new ProjectDatasetINodeCache(mLRUCap);
    
    mElasticSearch = new ElasticSearch(mElasticAddr, mElasticIndex, mElastticProjectType,
            mElasticDatasetType, mElasticInodeType, mElasticIssueTime, mElasticBatchsize, mStats);
    
    Ndb* mutations_tailer_connection = create_ndb_connection(mDatabaseName);
    mFsMutationsTableTailer = new FsMutationsTableTailer(mutations_tailer_connection, mPollMaxTimeToWait, mPDICache);
    
    MConn* mutations_connections = new MConn[mMutationsTU.mNumReaders];
    for(int i=0; i< mMutationsTU.mNumReaders; i++){
        mutations_connections[i].inodeConnection =  create_ndb_connection(mDatabaseName);
        mutations_connections[i].metadataConnection = create_ndb_connection(mMetaDatabaseName);
    }
    
    mFsMutationsDataReader = new FsMutationsDataReader(mutations_connections, mMutationsTU.mNumReaders, 
           mHopsworksEnabled, mElasticSearch, mPDICache, mLRUCap);
    mFsMutationsBatcher = new FsMutationsBatcher(mFsMutationsTableTailer, mFsMutationsDataReader, 
            mMutationsTU.mWaitTime, mMutationsTU.mBatchSize);
    

    if (mMetadataType == SchemaBased || mMetadataType == Both) {
        Ndb* metadata_tailer_connection = create_ndb_connection(mMetaDatabaseName);
        mMetadataTableTailer = new MetadataTableTailer(metadata_tailer_connection, mPollMaxTimeToWait);

        MConn* metadata_connections = new MConn[mMetadataTU.mNumReaders];
        for (int i = 0; i < mMetadataTU.mNumReaders; i++) {
            metadata_connections[i].inodeConnection = create_ndb_connection(mDatabaseName);
            metadata_connections[i].metadataConnection = create_ndb_connection(mMetaDatabaseName);
        }

        mMetadataReader = new MetadataReader(metadata_connections, mMetadataTU.mNumReaders,
                mHopsworksEnabled, mElasticSearch, mPDICache, mLRUCap);
        mMetadataBatcher = new MetadataBatcher(mMetadataTableTailer, mMetadataReader,
                mMetadataTU.mWaitTime, mMetadataTU.mBatchSize);
    }

    if (mMetadataType == Schemaless || mMetadataType == Both) {
        Ndb* s_metadata_tailer_connection = create_ndb_connection(mMetaDatabaseName);
        mSchemalessMetadataTailer = new SchemalessMetadataTailer(s_metadata_tailer_connection, mPollMaxTimeToWait);

        MConn* s_metadata_connections = new MConn[mSchemalessTU.mNumReaders];
        for (int i = 0; i < mSchemalessTU.mNumReaders; i++) {
            s_metadata_connections[i].inodeConnection = create_ndb_connection(mDatabaseName);
            s_metadata_connections[i].metadataConnection = create_ndb_connection(mMetaDatabaseName);
        }

        mSchemalessMetadataReader = new SchemalessMetadataReader(s_metadata_connections, mSchemalessTU.mNumReaders,
                mHopsworksEnabled, mElasticSearch, mPDICache);
        mSchemalessMetadataBatcher = new SchemalessMetadataBatcher(mSchemalessMetadataTailer,
                mSchemalessMetadataReader, mSchemalessTU.mWaitTime, mSchemalessTU.mBatchSize);
    }
    
    if (mHopsworksEnabled) {
        Ndb* project_tailer_connection = create_ndb_connection(mMetaDatabaseName);
        mProjectTableTailer = new ProjectTableTailer(project_tailer_connection, mPollMaxTimeToWait,
                mElasticSearch, mPDICache);
        
        Ndb* dataset_tailer_connection = create_ndb_connection(mMetaDatabaseName);
        mDatasetTableTailer = new DatasetTableTailer(dataset_tailer_connection, mPollMaxTimeToWait,
                mElasticSearch, mPDICache);
    }
    
}