int recursiveDelete(const char* root){ struct String_vector children; int i; int rc=zoo_get_children(zh,root,0,&children); if(rc!=ZNONODE){ if(rc!=ZOK){ LOG_ERROR(LOGSTREAM, "Failed to get children of %s, rc=%d",root,rc); return rc; } for(i=0;i<children.count; i++){ int rc = 0; char nodeName[2048]; snprintf(nodeName, sizeof(nodeName),"%s/%s",root,children.data[i]); rc=recursiveDelete(nodeName); if(rc!=ZOK){ free_String_vector(&children); return rc; } } free_String_vector(&children); } if(deletedCounter%1000==0) LOG_INFO(LOGSTREAM, "Deleting %s",root); rc=zoo_delete(zh,root,-1); if(rc!=ZOK){ LOG_ERROR(LOGSTREAM, "Failed to delete znode %s, rc=%d",root,rc); }else deletedCounter++; return rc; }
int main(int argc, char *argv[], char *envp[]) { INITSRVRTRC CEE_status sts = CEE_SUCCESS; SRVR_INIT_PARAM_Def initParam; DWORD processId; char tmpString[128]; char tmpString2[32]; char tmpString3[512]; CEECFG_Transport transport; CEECFG_TcpPortNumber portNumber; BOOL retcode; IDL_OBJECT_def srvrObjRef; CEECFG_TcpProcessName TcpProcessName; int TransportTrace = 0; CALL_COMP_DOVERS(ndcs,argc,argv); try { regZnodeName[0] = '\x0'; zkHost[0] = '\x0'; zkRootNode[0] = '\x0'; // Initialize seabed int sbResult; char buffer[FILENAME_MAX] = {0}; bzero(buffer, sizeof(buffer)); sbResult = file_init_attach(&argc, &argv, true, buffer); if(sbResult != XZFIL_ERR_OK){ exit(3); } sbResult = file_mon_process_startup(true); if(sbResult != XZFIL_ERR_OK){ exit(3); } msg_mon_enable_mon_messages(true); } catch(SB_Fatal_Excep sbfe) { exit(3); } sigset_t newset, oldset; sigemptyset(&newset); sigaddset(&newset,SIGQUIT); sigaddset(&newset,SIGTERM); sigprocmask(SIG_BLOCK,&newset,&oldset); processId = GetCurrentProcessId(); retcode = getInitParamSrvr(argc, argv, initParam, tmpString, tmpString3); retcode = TRUE; mxosrvr_init_seabed_trace_dll(); atexit(mxosrvr_atexit_function); // +++ Todo: Duplicating calls here. Should try to persist in srvrGlobal MS_Mon_Process_Info_Type proc_info; msg_mon_get_process_info_detail(NULL, &proc_info); myNid = proc_info.nid; myPid = proc_info.pid; myProcName = proc_info.process_name; char logNameSuffix[32]; sprintf( logNameSuffix, "_%d_%d.log", myNid, myPid ); CommonLogger::instance().initLog4cxx("log4cxx.trafodion.masterexe.config", logNameSuffix); if(retcode == FALSE ) { //LCOV_EXCL_START SendEventMsg( MSG_SET_SRVR_CONTEXT_FAILED, EVENTLOG_ERROR_TYPE, processId, ODBCMX_SERVER, srvrObjRef, 2, tmpString, tmpString3); exit(0); //LCOV_EXCL_STOP } GTransport.initialize(); if(GTransport.error != 0 ) { //LCOV_EXCL_START SendEventMsg( MSG_SET_SRVR_CONTEXT_FAILED, EVENTLOG_ERROR_TYPE, processId, ODBCMX_SERVER, srvrObjRef, 1, GTransport.error_message); exit(0); //LCOV_EXCL_STOP } chdir(GTransport.myPathname); initParam.srvrType = CORE_SRVR; //LCOV_EXCL_START if (initParam.debugFlag & SRVR_DEBUG_BREAK) { volatile int done = 0; while (!done) { sleep(10); } } //LCOV_EXCL_STOP char zkErrStr[2048]; stringstream zk_ip_port; // zoo_set_debug_level(ZOO_LOG_LEVEL_DEBUG); if( zkHost[0] == '\x0' && regZnodeName[0] == '\x0' ) { sprintf(zkErrStr, "***** Cannot get Zookeeper properties or registered znode info from startup params"); SendEventMsg( MSG_SET_SRVR_CONTEXT_FAILED, EVENTLOG_ERROR_TYPE, processId, ODBCMX_SERVER, srvrObjRef, 1, zkErrStr); // exit(1); } else { zk_ip_port << zkHost; sprintf(zkErrStr, "zk_ip_port is: %s", zk_ip_port.str().c_str()); SendEventMsg(MSG_SERVER_TRACE_INFO, EVENTLOG_INFORMATION_TYPE, processId, ODBCMX_SERVER, srvrObjRef, 1, zkErrStr); } if (initParam.debugFlag & SRVR_DEBUG_BREAK) zkSessionTimeout = 600; zoo_deterministic_conn_order(1); // enable deterministic order zh = zookeeper_init(zk_ip_port.str().c_str(), watcher, zkSessionTimeout * 1000, &myid, 0, 0); if (zh == 0){ sprintf(zkErrStr, "***** zookeeper_init() failed for host:port %s",zk_ip_port.str().c_str()); SendEventMsg( MSG_SET_SRVR_CONTEXT_FAILED, EVENTLOG_ERROR_TYPE, processId, ODBCMX_SERVER, srvrObjRef, 1, zkErrStr); // exit(1); } bool found = false; int rc; stringstream ss; ss.str(""); ss << zkRootNode << "/dcs/master"; string dcsMaster(ss.str()); Stat stat; int startPortNum = 0, portRangeNum; char masterHostName[MAX_HOST_NAME_LEN]; char startPort[12], portRange[12], masterTS[24]; struct String_vector children; children.count = 0; children.data = NULL; // Get the instance ID from registered node char *tkn; char tmpStr[256]; strcpy( tmpStr, regZnodeName ); tkn = strtok(tmpStr, ":" ); if(tkn!=NULL) strcpy(hostname,tkn); tkn = strtok(NULL, ":" ); if( tkn != NULL ) strcpy( instanceId, tkn ); tkn = strtok(NULL, ":" ); if( tkn != NULL ) strcpy( childId, tkn ); else ; // +++ Todo handle error while(!found) { rc = zoo_exists(zh, dcsMaster.c_str(), 0, &stat); if( rc == ZNONODE ) continue; else if( rc == ZOK ) { rc = zoo_get_children(zh, dcsMaster.c_str(), 0, &children); if( children.count > 0 ) { char zknodeName[2048]; strcpy(zknodeName, children.data[0]); tkn = strtok(zknodeName, ":" ); if( tkn != NULL ) strcpy( masterHostName, tkn ); tkn = strtok(NULL, ":" ); if( tkn != NULL ) { strcpy( startPort, tkn ); startPortNum = atoi(tkn); } tkn = strtok(NULL, ":" ); if( tkn != NULL ) { strcpy( portRange, tkn ); portRangeNum = atoi(tkn); } tkn = strtok(NULL, ":" ); if( tkn != NULL ) strcpy( masterTS, tkn ); free_String_vector(&children); found = true; } else continue; } else // error { sprintf(zkErrStr, "***** zoo_exists() for %s failed with error %d",dcsMaster.c_str(), rc); SendEventMsg( MSG_SET_SRVR_CONTEXT_FAILED, EVENTLOG_ERROR_TYPE, processId, ODBCMX_SERVER, srvrObjRef, 1, zkErrStr); break; } } // Initialize initparam to defaults initParam.transport = CEE_TRANSPORT_TCP; // -T 3 initParam.majorVersion = 3; // -V 3 // Will need to remove $ZTC0 and NonStopODBC from below sprintf( initParam.asSrvrObjRef, "TCP:$ZTC0/%s:NonStopODBC", startPort); // -A TCP:$ZTC0/52500:NonStopODBC // Will need to remove this after we get rid off all existing AS related processing sprintf( initParam.ASProcessName, "$MXOAS" ); // -AS $MXOAS // Will need to remove this after we get rid off all existing WMS related processing sprintf( initParam.QSProcessName, "$ZWMGR" ); // -QS $ZWMGR // moved this here from begining of the function BUILD_OBJECTREF(initParam.asSrvrObjRef, srvrObjRef, "NonStopODBC", initParam.portNumber); ss.str(""); ss << zkRootNode << "/dcs/servers/registered"; string dcsRegistered(ss.str()); char realpath[1024]; bool zk_error = false; if( found ) { sprintf(zkErrStr, "Found master node in Zookeeper"); SendEventMsg(MSG_SERVER_TRACE_INFO, EVENTLOG_INFORMATION_TYPE, processId, ODBCMX_SERVER, srvrObjRef, 1, zkErrStr); found = false; while(!found) { rc = zoo_exists(zh, dcsRegistered.c_str(), 0, &stat); if( rc == ZNONODE ) continue; else if( rc == ZOK ) { int i; //This section is the original port finding mechanism. //All servers (the herd) start looking for any available port //between starting port number+2 through port range max. //This is mainly for backward compatability for DcsServers //that don't pass PORTMAPTOSECS and PORTBINDTOSECS param if(portMapToSecs == -1 && portBindToSecs == -1) { for(i = startPortNum+2; i < startPortNum+portRangeNum; i++) { if (GTransport.m_listener->verifyPortAvailable("SRVR", i)) break; } if( i == startPortNum+portRangeNum ) { zk_error = true; sprintf(zkErrStr, "***** No ports free"); break; } } else { //This section is for new port map params, PORTMAPTOSECS and PORTBINDTOSECS, //passed in by DcsServer. DcsMaster writes the port map to data portion of //<username>/dcs/servers/registered znode. Wait PORTMAPTOSECS for port map //to appear in registered znode. When it appears read it and scan looking for //match of instance and child Id. long retryTimeout = 500;//.5 second long long timeout = JULIANTIMESTAMP(); bool isPortsMapped = false; char *zkData = new char[1000000]; int zkDataLen = 1000000; while(! isPortsMapped) { memset(zkData,0,1000000); rc = zoo_get(zh, dcsRegistered.c_str(), false, zkData, &zkDataLen, &stat); if( rc == ZOK && zkDataLen > 0 ) { sprintf(zkErrStr, "DCS port map = %s", zkData); SendEventMsg(MSG_SERVER_TRACE_INFO, EVENTLOG_INFORMATION_TYPE, processId, ODBCMX_SERVER, srvrObjRef, 1, zkErrStr); int myInstanceId = atoi(instanceId); int myChildId = atoi(childId); sprintf(zkErrStr, "Searching for my id (%d:%d) in port map",myInstanceId,myChildId); SendEventMsg(MSG_SERVER_TRACE_INFO, EVENTLOG_INFORMATION_TYPE, processId, ODBCMX_SERVER, srvrObjRef, 1, zkErrStr); char portMapInstanceId[8]; char portMapChildId[8]; char portMapPortNum[8]; char* saveptr; char* token = strtok_r (zkData,":",&saveptr); while (token != NULL) { if( token != NULL )//instance Id strcpy( portMapInstanceId, token ); token = strtok_r(NULL, ":",&saveptr); if( token != NULL )//child id strcpy( portMapChildId, token ); token = strtok_r(NULL, ":",&saveptr); if( token != NULL )//port number strcpy( portMapPortNum, token ); int currPortMapInstanceId = atoi(portMapInstanceId); int currPortMapChildId = atoi(portMapChildId); int currPortMapPortNum = atoi(portMapPortNum); if(myInstanceId == currPortMapInstanceId && myChildId == currPortMapChildId) { i = currPortMapPortNum; sprintf(zkErrStr, "Found my port number = %d in port map", i); SendEventMsg(MSG_SERVER_TRACE_INFO, EVENTLOG_INFORMATION_TYPE, processId, ODBCMX_SERVER, srvrObjRef, 1, zkErrStr); break; } else { token = strtok_r (NULL, ":",&saveptr); } } timeout = JULIANTIMESTAMP(); bool isAvailable = false; while ( isAvailable == false ) { if (GTransport.m_listener->verifyPortAvailable("SRVR", i)) { isAvailable = true; } else { if((JULIANTIMESTAMP() - timeout) > (portBindToSecs * 1000000)) { sprintf(zkErrStr, "Port bind timeout...exiting"); zk_error = true; break; } else { sprintf(zkErrStr, "Port = %d is already in use...retrying", i); SendEventMsg(MSG_SERVER_TRACE_INFO, EVENTLOG_INFORMATION_TYPE, processId, ODBCMX_SERVER, srvrObjRef, 1, zkErrStr); DELAY(retryTimeout); } } } isPortsMapped = true; } else { if((JULIANTIMESTAMP() - timeout) > (portMapToSecs * 1000000)) { sprintf(zkErrStr, "Port map read timeout...exiting"); zk_error = true; break; } else { sprintf(zkErrStr, "Waiting for port map"); SendEventMsg(MSG_SERVER_TRACE_INFO, EVENTLOG_INFORMATION_TYPE, processId, ODBCMX_SERVER, srvrObjRef, 1, zkErrStr); DELAY(retryTimeout); rc = zoo_exists(zh, dcsRegistered.c_str(), 0, &stat); } } } delete[] zkData; } initParam.portNumber = i; stringstream newpath; newpath.str(""); newpath << dcsRegistered.c_str() << "/" << regZnodeName; // dcsRegisteredNode.str(""); // dcsRegisteredNode << dcsRegistered.c_str() << "/" << regZnodeName; dcsRegisteredNode = newpath.str(); ss.str(""); ss << myPid; string pid(ss.str()); ss.str(""); ss << "STARTING" << ":" << JULIANTIMESTAMP() << ":" << ":" // Dialogue ID << myNid << ":" << myPid << ":" << myProcName.c_str() << ":" // Server IP address << ":" // Server Port << ":" // Client computer name << ":" // Client address << ":" // Client port << ":" // Client Appl name << ":"; regSrvrData = ss.str(); rc = zoo_create(zh, dcsRegisteredNode.c_str(), regSrvrData.c_str(), regSrvrData.length(), &ZOO_OPEN_ACL_UNSAFE, ZOO_EPHEMERAL, realpath, sizeof(realpath)-1); if( rc != ZOK ) { zk_error = true; sprintf(zkErrStr, "***** zoo_create() failed with error %d", rc); break; } found = true; } else // error { zk_error = true; sprintf(zkErrStr, "***** zoo_exists() for %s failed with error %d",dcsRegistered.c_str(), rc); break; } } } if( zk_error ) { SendEventMsg( MSG_SET_SRVR_CONTEXT_FAILED, EVENTLOG_ERROR_TYPE, processId, ODBCMX_SERVER, srvrObjRef, 1, zkErrStr); exit(1); } //LCOV_EXCL_START // when a server dies, the MXOAS sends message to CFG. CFG creates the MXOSRVR process // and passess only one command line atribute: -SQL CLEANUP OBSOLETE VOLATILE TABLES // It is for cleanup resources (volatile tables). // Newly created MXOSRVR process executes CLEANUP OBSOLETE VOLATILE TABLES and exits. // (This process is not managed by AS!. It is only a helper. if (initParam.sql != NULL) { if (strncmp(initParam.sql, "SELECT COUNT", 12) == 0) { //You can specify a completion code with any positive value in a PROCESS_STOP_. //Negative completion codes are reserved for HP use. //Therefore negative codes will return as 1000 + abs(completionCode) short completionCode = -1; completionCode = SQL_EXECDIRECT_FETCH(&initParam); if (completionCode < 0) completionCode = 1000 + abs(completionCode); #ifdef NSK_PLATFORM PROCESS_STOP_(,,,completionCode,,,,); #else /* * TODO: * need to revisit this logic to return a value via exit code * */ #endif } else {
/** * the main code that does the zookeeper leader * election. this code creates its own ephemeral * node on the given path and sees if its the first * one on the list and claims to be a leader if and only * if its the first one of children in the paretn path */ static int zkr_lock_operation(zkr_lock_mutex_t *mutex, struct timespec *ts) { zhandle_t *zh = mutex->zh; char *path = mutex->path; char *id = mutex->id; struct Stat stat; char* owner_id = NULL; int retry = 3; do { const clientid_t *cid = zoo_client_id(zh); // get the session id int64_t session = cid->client_id; char prefix[30]; int ret = 0; #if defined(__x86_64__) snprintf(prefix, 30, "x-%016lx-", session); #else snprintf(prefix, 30, "x-%016llx-", session); #endif struct String_vector vectorst; vectorst.data = NULL; vectorst.count = 0; ret = ZCONNECTIONLOSS; ret = retry_getchildren(zh, path, &vectorst, ts, retry); if (ret != ZOK) return ret; struct String_vector *vector = &vectorst; mutex->id = lookupnode(vector, prefix); free_String_vector(vector); if (mutex->id == NULL) { int len = strlen(path) + strlen(prefix) + 2; char buf[len]; char retbuf[len+20]; snprintf(buf, len, "%s/%s", path, prefix); ret = ZCONNECTIONLOSS; ret = zoo_create(zh, buf, NULL, 0, mutex->acl, ZOO_EPHEMERAL|ZOO_SEQUENCE, retbuf, (len+20)); // do not want to retry the create since // we would end up creating more than one child if (ret != ZOK) { LOG_WARN(("could not create zoo node %s", buf)); return ret; } mutex->id = getName(retbuf); } if (mutex->id != NULL) { ret = ZCONNECTIONLOSS; ret = retry_getchildren(zh, path, vector, ts, retry); if (ret != ZOK) { LOG_WARN(("could not connect to server")); return ret; } //sort this list sort_children(vector); owner_id = vector->data[0]; mutex->ownerid = strdup(owner_id); id = mutex->id; char* lessthanme = child_floor(vector->data, vector->count, id); if (lessthanme != NULL) { int flen = strlen(mutex->path) + strlen(lessthanme) + 2; char last_child[flen]; sprintf(last_child, "%s/%s",mutex->path, lessthanme); ret = ZCONNECTIONLOSS; LOG_WARN(("retry-exists-1")); ret = retry_zoowexists(zh, last_child, &lock_watcher_fn, mutex, &stat, ts, retry); LOG_WARN(("retry-exists-2")); // cannot watch my predecessor i am giving up // we need to be able to watch the predecessor // since if we do not become a leader the others // will keep waiting if (ret != ZOK) { free_String_vector(vector); LOG_WARN(("unable to watch my predecessor")); ret = zkr_lock_unlock(mutex); while (ret == 0) { //we have to give up our leadership // since we cannot watch out predecessor ret = zkr_lock_unlock(mutex); } return ret; } // we are not the owner of the lock mutex->isOwner = 0; } else { // this is the case when we are the owner // of the lock if (strcmp(mutex->id, owner_id) == 0) { LOG_DEBUG(("got the zoo lock owner - %s", mutex->id)); mutex->isOwner = 1; if (mutex->completion != NULL) { mutex->completion(0, mutex->cbdata); } return ZOK; } } free_String_vector(vector); return ZOK; } } while (mutex->id == NULL); return ZOK; }