int recursiveDelete(const char* root){
struct String_vector children;
int i;
int rc=zoo_get_children(zh,root,0,&children);
if(rc!=ZNONODE){
if(rc!=ZOK){
LOG_ERROR(LOGSTREAM, "Failed to get children of %s, rc=%d",root,rc);
return rc;
}
for(i=0;i<children.count; i++){
int rc = 0;
char nodeName[2048];
snprintf(nodeName, sizeof(nodeName),"%s/%s",root,children.data[i]);
rc=recursiveDelete(nodeName);
if(rc!=ZOK){
free_String_vector(&children);
return rc;
}
}
free_String_vector(&children);
}
if(deletedCounter%1000==0)
LOG_INFO(LOGSTREAM, "Deleting %s",root);
rc=zoo_delete(zh,root,-1);
if(rc!=ZOK){
LOG_ERROR(LOGSTREAM, "Failed to delete znode %s, rc=%d",root,rc);
}else
deletedCounter++;
return rc;
}
int main(int argc, char *argv[], char *envp[])
{
INITSRVRTRC
CEE_status sts = CEE_SUCCESS;
SRVR_INIT_PARAM_Def initParam;
DWORD processId;
char tmpString[128];
char tmpString2[32];
char tmpString3[512];
CEECFG_Transport transport;
CEECFG_TcpPortNumber portNumber;
BOOL retcode;
IDL_OBJECT_def srvrObjRef;
CEECFG_TcpProcessName TcpProcessName;
int TransportTrace = 0;
CALL_COMP_DOVERS(ndcs,argc,argv);
try
{
regZnodeName[0] = '\x0';
zkHost[0] = '\x0';
zkRootNode[0] = '\x0';
// Initialize seabed
int sbResult;
char buffer[FILENAME_MAX] = {0};
bzero(buffer, sizeof(buffer));
sbResult = file_init_attach(&argc, &argv, true, buffer);
if(sbResult != XZFIL_ERR_OK){
exit(3);
}
sbResult = file_mon_process_startup(true);
if(sbResult != XZFIL_ERR_OK){
exit(3);
}
msg_mon_enable_mon_messages(true);
}
catch(SB_Fatal_Excep sbfe)
{
exit(3);
}
sigset_t newset, oldset;
sigemptyset(&newset);
sigaddset(&newset,SIGQUIT);
sigaddset(&newset,SIGTERM);
sigprocmask(SIG_BLOCK,&newset,&oldset);
processId = GetCurrentProcessId();
retcode = getInitParamSrvr(argc, argv, initParam, tmpString, tmpString3);
retcode = TRUE;
mxosrvr_init_seabed_trace_dll();
atexit(mxosrvr_atexit_function);
// +++ Todo: Duplicating calls here. Should try to persist in srvrGlobal
MS_Mon_Process_Info_Type proc_info;
msg_mon_get_process_info_detail(NULL, &proc_info);
myNid = proc_info.nid;
myPid = proc_info.pid;
myProcName = proc_info.process_name;
char logNameSuffix[32];
sprintf( logNameSuffix, "_%d_%d.log", myNid, myPid );
CommonLogger::instance().initLog4cxx("log4cxx.trafodion.masterexe.config", logNameSuffix);
if(retcode == FALSE )
{
//LCOV_EXCL_START
SendEventMsg( MSG_SET_SRVR_CONTEXT_FAILED,
EVENTLOG_ERROR_TYPE,
processId,
ODBCMX_SERVER,
srvrObjRef,
2,
tmpString,
tmpString3);
exit(0);
//LCOV_EXCL_STOP
}
GTransport.initialize();
if(GTransport.error != 0 )
{
//LCOV_EXCL_START
SendEventMsg( MSG_SET_SRVR_CONTEXT_FAILED,
EVENTLOG_ERROR_TYPE,
processId,
ODBCMX_SERVER,
srvrObjRef,
1,
GTransport.error_message);
exit(0);
//LCOV_EXCL_STOP
}
chdir(GTransport.myPathname);
initParam.srvrType = CORE_SRVR;
//LCOV_EXCL_START
if (initParam.debugFlag & SRVR_DEBUG_BREAK)
{
volatile int done = 0;
while (!done) {
sleep(10);
}
}
//LCOV_EXCL_STOP
char zkErrStr[2048];
stringstream zk_ip_port;
// zoo_set_debug_level(ZOO_LOG_LEVEL_DEBUG);
if( zkHost[0] == '\x0' && regZnodeName[0] == '\x0' )
{
sprintf(zkErrStr, "***** Cannot get Zookeeper properties or registered znode info from startup params");
SendEventMsg( MSG_SET_SRVR_CONTEXT_FAILED,
EVENTLOG_ERROR_TYPE,
processId,
ODBCMX_SERVER,
srvrObjRef,
1,
zkErrStr);
// exit(1);
}
else
{
zk_ip_port << zkHost;
sprintf(zkErrStr, "zk_ip_port is: %s", zk_ip_port.str().c_str());
SendEventMsg(MSG_SERVER_TRACE_INFO, EVENTLOG_INFORMATION_TYPE,
processId, ODBCMX_SERVER,
srvrObjRef, 1, zkErrStr);
}
if (initParam.debugFlag & SRVR_DEBUG_BREAK)
zkSessionTimeout = 600;
zoo_deterministic_conn_order(1); // enable deterministic order
zh = zookeeper_init(zk_ip_port.str().c_str(), watcher, zkSessionTimeout * 1000, &myid, 0, 0);
if (zh == 0){
sprintf(zkErrStr, "***** zookeeper_init() failed for host:port %s",zk_ip_port.str().c_str());
SendEventMsg( MSG_SET_SRVR_CONTEXT_FAILED,
EVENTLOG_ERROR_TYPE,
processId,
ODBCMX_SERVER,
srvrObjRef,
1,
zkErrStr);
// exit(1);
}
bool found = false;
int rc;
stringstream ss;
ss.str("");
ss << zkRootNode << "/dcs/master";
string dcsMaster(ss.str());
Stat stat;
int startPortNum = 0, portRangeNum;
char masterHostName[MAX_HOST_NAME_LEN];
char startPort[12], portRange[12], masterTS[24];
struct String_vector children;
children.count = 0;
children.data = NULL;
// Get the instance ID from registered node
char *tkn;
char tmpStr[256];
strcpy( tmpStr, regZnodeName );
tkn = strtok(tmpStr, ":" );
if(tkn!=NULL)
strcpy(hostname,tkn);
tkn = strtok(NULL, ":" );
if( tkn != NULL )
strcpy( instanceId, tkn );
tkn = strtok(NULL, ":" );
if( tkn != NULL )
strcpy( childId, tkn );
else
; // +++ Todo handle error
while(!found)
{
rc = zoo_exists(zh, dcsMaster.c_str(), 0, &stat);
if( rc == ZNONODE )
continue;
else
if( rc == ZOK )
{
rc = zoo_get_children(zh, dcsMaster.c_str(), 0, &children);
if( children.count > 0 )
{
char zknodeName[2048];
strcpy(zknodeName, children.data[0]);
tkn = strtok(zknodeName, ":" );
if( tkn != NULL )
strcpy( masterHostName, tkn );
tkn = strtok(NULL, ":" );
if( tkn != NULL ) {
strcpy( startPort, tkn );
startPortNum = atoi(tkn);
}
tkn = strtok(NULL, ":" );
if( tkn != NULL ) {
strcpy( portRange, tkn );
portRangeNum = atoi(tkn);
}
tkn = strtok(NULL, ":" );
if( tkn != NULL )
strcpy( masterTS, tkn );
free_String_vector(&children);
found = true;
}
else
continue;
}
else // error
{
sprintf(zkErrStr, "***** zoo_exists() for %s failed with error %d",dcsMaster.c_str(), rc);
SendEventMsg( MSG_SET_SRVR_CONTEXT_FAILED,
EVENTLOG_ERROR_TYPE,
processId,
ODBCMX_SERVER,
srvrObjRef,
1,
zkErrStr);
break;
}
}
// Initialize initparam to defaults
initParam.transport = CEE_TRANSPORT_TCP; // -T 3
initParam.majorVersion = 3; // -V 3
// Will need to remove $ZTC0 and NonStopODBC from below
sprintf( initParam.asSrvrObjRef, "TCP:$ZTC0/%s:NonStopODBC", startPort); // -A TCP:$ZTC0/52500:NonStopODBC
// Will need to remove this after we get rid off all existing AS related processing
sprintf( initParam.ASProcessName, "$MXOAS" ); // -AS $MXOAS
// Will need to remove this after we get rid off all existing WMS related processing
sprintf( initParam.QSProcessName, "$ZWMGR" ); // -QS $ZWMGR
// moved this here from begining of the function
BUILD_OBJECTREF(initParam.asSrvrObjRef, srvrObjRef, "NonStopODBC", initParam.portNumber);
ss.str("");
ss << zkRootNode << "/dcs/servers/registered";
string dcsRegistered(ss.str());
char realpath[1024];
bool zk_error = false;
if( found )
{
sprintf(zkErrStr, "Found master node in Zookeeper");
SendEventMsg(MSG_SERVER_TRACE_INFO, EVENTLOG_INFORMATION_TYPE,
processId, ODBCMX_SERVER,
srvrObjRef, 1, zkErrStr);
found = false;
while(!found)
{
rc = zoo_exists(zh, dcsRegistered.c_str(), 0, &stat);
if( rc == ZNONODE )
continue;
else
if( rc == ZOK )
{
int i;
//This section is the original port finding mechanism.
//All servers (the herd) start looking for any available port
//between starting port number+2 through port range max.
//This is mainly for backward compatability for DcsServers
//that don't pass PORTMAPTOSECS and PORTBINDTOSECS param
if(portMapToSecs == -1 && portBindToSecs == -1) {
for(i = startPortNum+2; i < startPortNum+portRangeNum; i++) {
if (GTransport.m_listener->verifyPortAvailable("SRVR", i))
break;
}
if( i == startPortNum+portRangeNum )
{
zk_error = true;
sprintf(zkErrStr, "***** No ports free");
break;
}
} else {
//This section is for new port map params, PORTMAPTOSECS and PORTBINDTOSECS,
//passed in by DcsServer. DcsMaster writes the port map to data portion of
//<username>/dcs/servers/registered znode. Wait PORTMAPTOSECS for port map
//to appear in registered znode. When it appears read it and scan looking for
//match of instance and child Id.
long retryTimeout = 500;//.5 second
long long timeout = JULIANTIMESTAMP();
bool isPortsMapped = false;
char *zkData = new char[1000000];
int zkDataLen = 1000000;
while(! isPortsMapped) {
memset(zkData,0,1000000);
rc = zoo_get(zh, dcsRegistered.c_str(), false, zkData, &zkDataLen, &stat);
if( rc == ZOK && zkDataLen > 0 ) {
sprintf(zkErrStr, "DCS port map = %s", zkData);
SendEventMsg(MSG_SERVER_TRACE_INFO, EVENTLOG_INFORMATION_TYPE,
processId, ODBCMX_SERVER,
srvrObjRef, 1, zkErrStr);
int myInstanceId = atoi(instanceId);
int myChildId = atoi(childId);
sprintf(zkErrStr, "Searching for my id (%d:%d) in port map",myInstanceId,myChildId);
SendEventMsg(MSG_SERVER_TRACE_INFO, EVENTLOG_INFORMATION_TYPE,
processId, ODBCMX_SERVER,
srvrObjRef, 1, zkErrStr);
char portMapInstanceId[8];
char portMapChildId[8];
char portMapPortNum[8];
char* saveptr;
char* token = strtok_r (zkData,":",&saveptr);
while (token != NULL)
{
if( token != NULL )//instance Id
strcpy( portMapInstanceId, token );
token = strtok_r(NULL, ":",&saveptr);
if( token != NULL )//child id
strcpy( portMapChildId, token );
token = strtok_r(NULL, ":",&saveptr);
if( token != NULL )//port number
strcpy( portMapPortNum, token );
int currPortMapInstanceId = atoi(portMapInstanceId);
int currPortMapChildId = atoi(portMapChildId);
int currPortMapPortNum = atoi(portMapPortNum);
if(myInstanceId == currPortMapInstanceId && myChildId == currPortMapChildId) {
i = currPortMapPortNum;
sprintf(zkErrStr, "Found my port number = %d in port map", i);
SendEventMsg(MSG_SERVER_TRACE_INFO, EVENTLOG_INFORMATION_TYPE,
processId, ODBCMX_SERVER,
srvrObjRef, 1, zkErrStr);
break;
} else {
token = strtok_r (NULL, ":",&saveptr);
}
}
timeout = JULIANTIMESTAMP();
bool isAvailable = false;
while ( isAvailable == false ) {
if (GTransport.m_listener->verifyPortAvailable("SRVR", i)) {
isAvailable = true;
} else {
if((JULIANTIMESTAMP() - timeout) > (portBindToSecs * 1000000)) {
sprintf(zkErrStr, "Port bind timeout...exiting");
zk_error = true;
break;
} else {
sprintf(zkErrStr, "Port = %d is already in use...retrying", i);
SendEventMsg(MSG_SERVER_TRACE_INFO, EVENTLOG_INFORMATION_TYPE,
processId, ODBCMX_SERVER,
srvrObjRef, 1, zkErrStr);
DELAY(retryTimeout);
}
}
}
isPortsMapped = true;
} else {
if((JULIANTIMESTAMP() - timeout) > (portMapToSecs * 1000000)) {
sprintf(zkErrStr, "Port map read timeout...exiting");
zk_error = true;
break;
} else {
sprintf(zkErrStr, "Waiting for port map");
SendEventMsg(MSG_SERVER_TRACE_INFO, EVENTLOG_INFORMATION_TYPE,
processId, ODBCMX_SERVER,
srvrObjRef, 1, zkErrStr);
DELAY(retryTimeout);
rc = zoo_exists(zh, dcsRegistered.c_str(), 0, &stat);
}
}
}
delete[] zkData;
}
initParam.portNumber = i;
stringstream newpath;
newpath.str("");
newpath << dcsRegistered.c_str() << "/" << regZnodeName;
// dcsRegisteredNode.str("");
// dcsRegisteredNode << dcsRegistered.c_str() << "/" << regZnodeName;
dcsRegisteredNode = newpath.str();
ss.str("");
ss << myPid;
string pid(ss.str());
ss.str("");
ss << "STARTING"
<< ":"
<< JULIANTIMESTAMP()
<< ":"
<< ":" // Dialogue ID
<< myNid
<< ":"
<< myPid
<< ":"
<< myProcName.c_str()
<< ":" // Server IP address
<< ":" // Server Port
<< ":" // Client computer name
<< ":" // Client address
<< ":" // Client port
<< ":" // Client Appl name
<< ":";
regSrvrData = ss.str();
rc = zoo_create(zh, dcsRegisteredNode.c_str(), regSrvrData.c_str(), regSrvrData.length(), &ZOO_OPEN_ACL_UNSAFE, ZOO_EPHEMERAL, realpath, sizeof(realpath)-1);
if( rc != ZOK )
{
zk_error = true;
sprintf(zkErrStr, "***** zoo_create() failed with error %d", rc);
break;
}
found = true;
}
else // error
{
zk_error = true;
sprintf(zkErrStr, "***** zoo_exists() for %s failed with error %d",dcsRegistered.c_str(), rc);
break;
}
}
}
if( zk_error ) {
SendEventMsg( MSG_SET_SRVR_CONTEXT_FAILED,
EVENTLOG_ERROR_TYPE,
processId,
ODBCMX_SERVER,
srvrObjRef,
1,
zkErrStr);
exit(1);
}
//LCOV_EXCL_START
// when a server dies, the MXOAS sends message to CFG. CFG creates the MXOSRVR process
// and passess only one command line atribute: -SQL CLEANUP OBSOLETE VOLATILE TABLES
// It is for cleanup resources (volatile tables).
// Newly created MXOSRVR process executes CLEANUP OBSOLETE VOLATILE TABLES and exits.
// (This process is not managed by AS!. It is only a helper.
if (initParam.sql != NULL)
{
if (strncmp(initParam.sql, "SELECT COUNT", 12) == 0)
{
//You can specify a completion code with any positive value in a PROCESS_STOP_.
//Negative completion codes are reserved for HP use.
//Therefore negative codes will return as 1000 + abs(completionCode)
short completionCode = -1;
completionCode = SQL_EXECDIRECT_FETCH(&initParam);
if (completionCode < 0)
completionCode = 1000 + abs(completionCode);
#ifdef NSK_PLATFORM
PROCESS_STOP_(,,,completionCode,,,,);
#else
/*
* TODO:
* need to revisit this logic to return a value via exit code
*
*/
#endif
}
else
{
/**
* the main code that does the zookeeper leader
* election. this code creates its own ephemeral
* node on the given path and sees if its the first
* one on the list and claims to be a leader if and only
* if its the first one of children in the paretn path
*/
static int zkr_lock_operation(zkr_lock_mutex_t *mutex, struct timespec *ts) {
zhandle_t *zh = mutex->zh;
char *path = mutex->path;
char *id = mutex->id;
struct Stat stat;
char* owner_id = NULL;
int retry = 3;
do {
const clientid_t *cid = zoo_client_id(zh);
// get the session id
int64_t session = cid->client_id;
char prefix[30];
int ret = 0;
#if defined(__x86_64__)
snprintf(prefix, 30, "x-%016lx-", session);
#else
snprintf(prefix, 30, "x-%016llx-", session);
#endif
struct String_vector vectorst;
vectorst.data = NULL;
vectorst.count = 0;
ret = ZCONNECTIONLOSS;
ret = retry_getchildren(zh, path, &vectorst, ts, retry);
if (ret != ZOK)
return ret;
struct String_vector *vector = &vectorst;
mutex->id = lookupnode(vector, prefix);
free_String_vector(vector);
if (mutex->id == NULL) {
int len = strlen(path) + strlen(prefix) + 2;
char buf[len];
char retbuf[len+20];
snprintf(buf, len, "%s/%s", path, prefix);
ret = ZCONNECTIONLOSS;
ret = zoo_create(zh, buf, NULL, 0, mutex->acl,
ZOO_EPHEMERAL|ZOO_SEQUENCE, retbuf, (len+20));
// do not want to retry the create since
// we would end up creating more than one child
if (ret != ZOK) {
LOG_WARN(("could not create zoo node %s", buf));
return ret;
}
mutex->id = getName(retbuf);
}
if (mutex->id != NULL) {
ret = ZCONNECTIONLOSS;
ret = retry_getchildren(zh, path, vector, ts, retry);
if (ret != ZOK) {
LOG_WARN(("could not connect to server"));
return ret;
}
//sort this list
sort_children(vector);
owner_id = vector->data[0];
mutex->ownerid = strdup(owner_id);
id = mutex->id;
char* lessthanme = child_floor(vector->data, vector->count, id);
if (lessthanme != NULL) {
int flen = strlen(mutex->path) + strlen(lessthanme) + 2;
char last_child[flen];
sprintf(last_child, "%s/%s",mutex->path, lessthanme);
ret = ZCONNECTIONLOSS;
LOG_WARN(("retry-exists-1"));
ret = retry_zoowexists(zh, last_child, &lock_watcher_fn, mutex,
&stat, ts, retry);
LOG_WARN(("retry-exists-2"));
// cannot watch my predecessor i am giving up
// we need to be able to watch the predecessor
// since if we do not become a leader the others
// will keep waiting
if (ret != ZOK) {
free_String_vector(vector);
LOG_WARN(("unable to watch my predecessor"));
ret = zkr_lock_unlock(mutex);
while (ret == 0) {
//we have to give up our leadership
// since we cannot watch out predecessor
ret = zkr_lock_unlock(mutex);
}
return ret;
}
// we are not the owner of the lock
mutex->isOwner = 0;
}
else {
// this is the case when we are the owner
// of the lock
if (strcmp(mutex->id, owner_id) == 0) {
LOG_DEBUG(("got the zoo lock owner - %s", mutex->id));
mutex->isOwner = 1;
if (mutex->completion != NULL) {
mutex->completion(0, mutex->cbdata);
}
return ZOK;
}
}
free_String_vector(vector);
return ZOK;
}
} while (mutex->id == NULL);
return ZOK;
}
