IPropertyTree *getPkgSetRegistry(const char *process, bool readonly)
{
Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageSets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
if (!globalLock)
throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to connect to PackageSet information in dali /PackageSets");
IPropertyTree *pkgSets = globalLock->queryRoot();
if (!pkgSets)
throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to open PackageSet information in dali /PackageSets");
if (!process || !*process)
process = "*";
StringBuffer id;
buildPkgSetId(id, process);
//Only lock the branch for the target we're interested in.
VStringBuffer xpath("/PackageSets/PackageSet[@id='%s']", id.str());
Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
if (!conn)
{
if (readonly)
return NULL;
Owned<IPropertyTree> pkgSet = createPTree();
pkgSet->setProp("@id", id.str());
pkgSet->setProp("@process", process);
pkgSets->addPropTree("PackageSet", pkgSet.getClear());
globalLock->commit();
conn.setown(querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
}
return (conn) ? conn->getRoot() : NULL;
}
bool deletePkgInfo(const char *name, const char *target, const char *process, bool globalScope)
{
Owned<IRemoteConnection> pkgSetsConn = querySDS().connect("/PackageSets/", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
if (!pkgSetsConn)
throw MakeStringException(PKG_NONE_DEFINED, "No package sets defined");
IPropertyTree* packageSets = pkgSetsConn->queryRoot();
StringBuffer pkgSetId;
buildPkgSetId(pkgSetId, process);
VStringBuffer pkgSet_xpath("PackageSet[@id='%s']", pkgSetId.str());
IPropertyTree *pkgSetRegistry = packageSets->queryPropTree(pkgSet_xpath.str());
if (!pkgSetRegistry)
throw MakeStringException(PKG_TARGET_NOT_DEFINED, "No package sets defined for %s", process);
StringBuffer lcTarget(target);
target = lcTarget.toLowerCase().str();
StringBuffer lcName(name);
name = lcName.toLowerCase().str();
Owned<IPropertyTree> mapEntry;
StringBuffer xpath;
if (!globalScope)
{
xpath.appendf("PackageMap[@id='%s::%s'][@querySet='%s']", target, name, target);
mapEntry.setown(pkgSetRegistry->getPropTree(xpath.str()));
}
if (!mapEntry)
{
xpath.clear().appendf("PackageMap[@id='%s'][@querySet='%s']", name, target);
mapEntry.setown(pkgSetRegistry->getPropTree(xpath.str()));
if (!mapEntry)
throw MakeStringException(PKG_DELETE_NOT_FOUND, "Unable to delete %s - information not found", lcName.str());
}
StringAttr pmid(mapEntry->queryProp("@id"));
pkgSetRegistry->removeTree(mapEntry);
xpath.clear().appendf("PackageSet/PackageMap[@id='%s']", pmid.get());
if (!packageSets->hasProp(xpath))
{
Owned<IRemoteConnection> pkgMapsConn = querySDS().connect("/PackageMaps/", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
if (!pkgMapsConn)
throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to retrieve PackageMaps information from dali [/PackageMaps]");
IPropertyTree *pkgMaps = pkgMapsConn->queryRoot();
if (!pkgMaps)
throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to retrieve PackageMaps information from dali [/PackageMaps]");
IPropertyTree *mapTree = pkgMaps->queryPropTree(xpath.clear().appendf("PackageMap[@id='%s']", pmid.get()).str());
if (mapTree)
pkgMaps->removeTree(mapTree);
}
return true;
}
IRemoteConnection * getEntryConnection(const char * name, unsigned mode)
{
StringBuffer xpath;
getNewXPath(xpath, name);
Owned<IRemoteConnection> connection = querySDS().connect(xpath.str(), myProcessSession(), mode, CONNECTION_TIMEOUT);
if (connection)
return connection.getClear();
//Retain backwards compatibility for the moment
getOldXPath(xpath.clear(), name);
return querySDS().connect(xpath.str(), myProcessSession(), mode, CONNECTION_TIMEOUT);
}
bool isScheduledWorkUnit(char const * wuid)
{
Owned<IRemoteConnection> conn = querySDS().connect("/Schedule", myProcessSession(), RTM_LOCK_WRITE | RTM_CREATE_QUERY, connectionTimeout);
StringBuffer xpath("*/*/*/");
ncnameEscape(wuid, xpath);
return conn->queryRoot()->hasProp(xpath.str());
}
IFvDataSource * createRemoteFileDataSource(const SocketEndpoint & server, const char * username, const char * password, const char * logicalName)
{
Owned<INode> serverNode = createINode(server);
CMessageBuffer msg;
msg.setEndian(__BIG_ENDIAN);
msg.append((byte)FVCMDcreatefile);
msg.append(myProcessSession());
msg.append(username);
msg.append(password);
msg.append(logicalName);
sendReceive(serverNode, msg);
unsigned short version;
unique_id_t id;
__int64 numRows;
bool isIndex;
msg.read(version);
msg.read(id);
msg.read(numRows);
Owned<IFvDataSourceMetaData> meta = deserializeDataSourceMeta(msg);
msg.read(isIndex);
if (id)
return new RemoteDataSource(server, id, meta, numRows, isIndex);
return 0;
}
static void emptyScopes()
{
PROGLOG("Removing empty scopes");
Owned<IDFScopeIterator> iter = queryDistributedFileDirectory().getScopeIterator(NULL,true,true);
CDfsLogicalFileName dlfn;
StringBuffer s;
StringArray toremove;
ForEach(*iter) {
CDfsLogicalFileName dlfn;
StringBuffer scope;
scope.append(iter->query());
dlfn.set(scope.str(),"x");
dlfn.makeScopeQuery(s.clear(),true);
Owned<IRemoteConnection> conn = querySDS().connect(s.str(),myProcessSession(),RTM_LOCK_READ, INFINITE);
if (!conn)
DBGLOG("Could not connect to '%s' using %s",iter->query(),s.str());
else {
if (recursiveCheckEmptyScope(*conn->queryRoot())) {
PROGLOG("Empty scope %s",iter->query());
toremove.append(iter->query());
}
}
}
iter.clear();
ForEachItemIn(i,toremove) {
PROGLOG("Removed scope %s",toremove.item(i));
queryDistributedFileDirectory().removeEmptyScope(toremove.item(i));
}
static bool ensureThorIsDown(const char *cluster, bool nofail, bool wait)
{
bool retry = false;
do {
Owned<IRemoteConnection> pStatus = querySDS().connect("/Status/Servers", myProcessSession(), RTM_NONE, SDS_LOCK_TIMEOUT);
Owned<IPropertyTreeIterator> it = pStatus->queryRoot()->getElements("Server[@name='ThorMaster']");
retry = false;
ForEach(*it) {
IPropertyTree* pServer = &it->query();
if (pServer->hasProp("@cluster") && !strcmp(pServer->queryProp("@cluster"), cluster)) {
if (nofail) {
WARNLOG("A Thor on cluster %s is still active", cluster);
if (!wait)
return false;
Sleep(1000*10);
PROGLOG("Retrying...");
retry = true;
break;
}
throw MakeStringException(-1, "A Thor cluster node swap requires the cluster to be offline. Please stop the Thor cluster '%s' and try again.", cluster);
}
}
} while (retry);
return true;
}
IRemoteConnection* CWSESPControlEx::querySDSConnection(const char* xpath, unsigned mode, unsigned timeout)
{
Owned<IRemoteConnection> globalLock = querySDS().connect(xpath, myProcessSession(), mode, timeout);
if (!globalLock)
throw MakeStringException(ECLWATCH_INTERNAL_ERROR, "Unable to connect to ESP Session information in dali %s", xpath);
return globalLock.getClear();
}
COnlineWorkunitIterator (
IUserDescriptor* _user,
const char *_namelo,
const char *_namehi,
const char *user,
const char *cluster,
const char *jobname,
const char *state,
const char *priority,
const char *fileread,
const char *filewritten,
const char *roxiecluster,
const char *ecl
) : user(_user), namelo(_namelo), namehi(_namehi)
{
securitydisabled = false;
if (namelo.isEmpty())
namelo.set("W");
if (namehi.isEmpty()) {
StringBuffer tmp;
namehi.set(getWUIDdaysAgo(tmp,-1).str());
}
const char *lo = namelo;
const char *hi = namehi;
StringBuffer query;
while (*lo&&(toupper(*lo)==toupper(*hi))) {
query.append((char)toupper(*lo));
lo++;
hi++;
}
if (*lo||*hi)
query.append("*");
if (user&&*user)
query.appendf("[@submitID=~?\"%s\"]",user);
if (cluster&&*cluster)
query.appendf("[@clusterName=~?\"%s\"]",cluster);
if (jobname&&*jobname)
query.appendf("[@jobName=~?\"%s\"]",jobname);
if (state&&*state)
query.appendf("[@state=?\"%s\"]",state);
if (priority&&*priority)
query.appendf("[@priorityClass=?\"%s\"]",priority);
if (fileread&&*fileread)
query.appendf("[FilesRead/File/@name=~?\"%s\"]",fileread);
if (filewritten&&*filewritten)
query.appendf("[Files/File/@name=~?\"%s\"]",filewritten);
if (roxiecluster&&*roxiecluster)
query.appendf("[RoxieQueryInfo/@roxieClusterName=~?\"%s\"]",roxiecluster);
if (ecl&&*ecl)
query.appendf("[Query/Text=?~\"*%s*\"]",ecl);
conn.setown(querySDS().connect("WorkUnits", myProcessSession(), 0, SDS_LOCK_TIMEOUT));
if (conn.get()) {
iter.setown(conn->getElements(query.str()));
if (!iter.get())
conn.clear();
}
}
static unsigned fn2(unsigned n, unsigned m, unsigned seed, unsigned depth, StringBuffer &parentname)
{
if (!Rconn)
return 0;
if ((n+m+seed)%25==0) {
Rconn->commit();
Rconn->Release();
Rconn = querySDS().connect("/DAREGRESS",myProcessSession(), 0, 1000000);
if (!Rconn) {
ERROR("Failed to connect to /DAREGRESS");
return 0;
}
}
IPropertyTree *parent = parentname.length()?Rconn->queryRoot()->queryPropTree(parentname.str()):Rconn->queryRoot();
if (!parent) {
ERROR1("Failed to connect to %s",parentname.str());
Rconn->Release();
Rconn = NULL;
return 0;
}
__int64 val = parent->getPropInt64("val",0);
parent->setPropInt64("val",n+val);
val = parent->getPropInt64("@val",0);
parent->setPropInt64("@val",m+val);
val = parent->getPropInt64(NULL,0);
parent->setPropInt64(NULL,seed+val);
if (!seed)
return m+n;
if (n==m)
return seed;
if (depth>10)
return seed+n+m;
if (seed%7==n%7)
return n;
if (seed%7==m%7)
return m;
char name[64];
unsigned v = seed;
name[0] = 's';
name[1] = 'u';
name[2] = 'b';
unsigned i = 3;
while (v) {
name[i++] = ('A'+v%26 );
v /= 26;
}
name[i] = 0;
unsigned l = parentname.length();
if (parentname.length())
parentname.append('/');
parentname.append(name);
IPropertyTree *child = parent->queryPropTree(name);
if (!child)
child = parent->addPropTree(name, createPTree(name));
unsigned ret = fn2(fn2(n,seed,seed*17+11,depth+1,parentname),fn2(seed,m,seed*11+17,depth+1,parentname),seed*19+7,depth+1,parentname);
parentname.setLength(l);
return ret;
}
void cleanupWorkUnitSchedule()
{
Owned<IRemoteConnection> conn = querySDS().connect("/Schedule", myProcessSession(), RTM_LOCK_WRITE, connectionTimeout);
if(!conn) return;
Owned<IPropertyTree> root(conn->queryRoot()->getBranch("."));
recursiveCleanup(root, 2);
conn->commit();
cleanupSchedulerList(root);
}
void init()
{
VStringBuffer xpath("PackageMap[@id='%s']", pmid.str());
globalLock.setown(querySDS().connect("/PackageMaps", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT));
packageMaps = globalLock->queryRoot();
pmExisting = packageMaps->queryPropTree(xpath);
if (pmExisting && !pmExisting->getPropBool("@multipart", false))
convertExisting();
}
IXRefNode * CXRefNodeManager::CreateXRefNode(const char* NodeName)
{
Owned<IRemoteConnection> conn = querySDS().connect("/DFU/XREF",myProcessSession(),RTM_CREATE_QUERY|RTM_LOCK_WRITE ,INFINITE);
IPropertyTree* xref_ptree = conn->queryRoot();
IPropertyTree* cluster_ptree = xref_ptree->addPropTree("Cluster", createPTree());
cluster_ptree->setProp("@name",NodeName);
conn->commit();
conn->changeMode(RTM_NONE);
return new CXRefNode(NodeName,conn);
}
void cleanupSchedulerList(IPropertyTree * schedule)
{
Owned<IRemoteConnection> conn = querySDS().connect("/Schedulers", myProcessSession(), RTM_LOCK_WRITE, connectionTimeout);
if(!conn) return;
Owned<IPropertyTree> root(conn->queryRoot()->getBranch("."));
Owned<IPropertyTreeIterator> iter = root->getElements("*");
for(iter->first(); iter->isValid(); iter->next())
if(!schedule->hasProp(iter->query().queryName()))
iter->query().setProp("@remove", "yes");
bool more;
do more = root->removeProp("*[@remove=\"yes\"]"); while(more);
}
void descheduleNonexistentWorkUnit(char const * wuid)
{
StringBuffer xpath;
xpath.append("*/*/*/");
ncnameEscape(wuid, xpath);
Owned<IRemoteConnection> conn = querySDS().connect("/Schedule", myProcessSession(), RTM_LOCK_WRITE, connectionTimeout);
if(!conn) return;
PROGLOG("Scheduled workunit %s could not be found, and so is being descheduled", wuid);
Owned<IPropertyTree> root = conn->getRoot();
bool more;
do more = root->removeProp(xpath.str()); while(more);
}
void PackageMapAndSet::load(const char* path, IPropertyTree* t)
{
Owned<IRemoteConnection> globalLock = querySDS().connect(path, myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
if (!globalLock)
throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to retrieve %s information from dali /%s", path, path);
t->removeProp(path);
IPropertyTree *root = globalLock->queryRoot();
if (root)
t->addPropTree(path, createPTreeFromIPT(root));
}
void findUnusedFilesInDFS(StringArray &unusedFiles, const char *process, const MapStringTo<bool> &usedFileMap)
{
Owned<IRemoteConnection> globalLock = querySDS().connect("/Files/", myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
Owned<IPropertyTree> root = globalLock->getRoot();
VStringBuffer xpath("//File[Cluster/@name='%s']/OrigName", process);
Owned<IPropertyTreeIterator> files = root->getElements(xpath);
ForEach(*files)
{
const char *lfn = skipTilda(files->query().queryProp(NULL));
if (lfn && !usedFileMap.getValue(lfn))
unusedFiles.append(lfn);
}
}
bool getAllClusters(SocketEndpointArray &eps)
{
Owned<IRemoteConnection> conn = querySDS().connect("/Environment/Software", myProcessSession(), RTM_LOCK_READ, SDS_CONNECT_TIMEOUT);
if (!conn)
return false;
IPropertyTree* root = conn->queryRoot();
Owned<IPropertyTreeIterator> clusters= root->getElements("ThorCluster");
if (clusters->first()) {
do {
IPropertyTree &cluster = clusters->query();
if (!getCluster(cluster.queryProp("@name"),eps))
ERRLOG("Cluster %s not found",cluster.queryProp("@name"));
} while (clusters->next());
}
return eps.ordinality()!=0;
}
bool initClientProcess(IGroup *servergrp, DaliClientRole role, unsigned mpport, const char *clientVersion, const char *minServerVersion, unsigned timeout)
{
assertex(servergrp);
daliClientIsActive = true;
startMPServer(mpport);
Owned<ICommunicator> comm(createCommunicator(servergrp,true));
IGroup * covengrp;
if (!registerClientProcess(comm.get(),covengrp,timeout,role))
{
daliClientIsActive = false;
return false;
}
initCoven(covengrp,NULL,clientVersion, minServerVersion);
covengrp->Release();
queryLogMsgManager()->setSession(myProcessSession());
return true;
}
CSDSServerStatus::CSDSServerStatus(const char *servername)
{
conn = querySDS().connect("Status/Servers/Server", myProcessSession(), RTM_CREATE_ADD | RTM_LOCK_READ | RTM_DELETE_ON_DISCONNECT, 5*60*1000);
if (conn) {
IPropertyTree &root = *conn->queryRoot();
root.setProp("@name",servername);
StringBuffer node;
queryMyNode()->endpoint().getIpText(node);
root.setProp("@node",node.str());
root.setPropInt("@mpport",queryMyNode()->endpoint().port);
CDateTime dt;
dt.setNow();
StringBuffer str;
root.setProp("@started",dt.getString(str).str());
conn->commit();
}
}
void init()
{
StringBuffer xpath("Software/ThorCluster[@name=\"");
xpath.append(clusterName).append("\"]");
Owned<IRemoteConnection> conn = querySDS().connect("/Environment", myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
environment.setown(createPTreeFromIPT(conn->queryRoot()));
options = environment->queryPropTree(xpath.str());
if (!options)
throwUnexpected();
groupName.set(options->queryProp("@nodeGroup"));
if (groupName.isEmpty())
groupName.set(options->queryProp("@name"));
VStringBuffer spareS("%s_spares", groupName.get());
spareGroupName.set(spareS);
group.setown(queryNamedGroupStore().lookup(groupName));
spareGroup.setown(queryNamedGroupStore().lookup(spareGroupName));
}
void activatePackageMapInfo(const char *target, const char *name, const char *process, bool globalScope, bool activate)
{
if (!target || !*target)
throw MakeStringExceptionDirect(PKG_TARGET_NOT_DEFINED, "No target defined");
if (!name || !*name)
throw MakeStringExceptionDirect(PKG_MISSING_PARAM, "No pmid specified");
Owned<IRemoteConnection> globalLock = querySDS().connect("PackageSets", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
if (!globalLock)
throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to retrieve PackageSets information from dali /PackageSets");
StringBuffer lcTarget(target);
target = lcTarget.toLowerCase().str();
StringBuffer lcName(name);
name = lcName.toLowerCase().str();
IPropertyTree *root = globalLock->queryRoot();
if (!root)
throw MakeStringException(PKG_ACTIVATE_NOT_FOUND, "Unable to retrieve PackageSet information");
StringBuffer pkgSetId;
buildPkgSetId(pkgSetId, process);
VStringBuffer xpath("PackageSet[@id='%s']", pkgSetId.str());
IPropertyTree *pkgSetTree = root->queryPropTree(xpath);
if (pkgSetTree)
{
IPropertyTree *mapTree = NULL;
if (!globalScope)
{
xpath.clear().appendf("PackageMap[@querySet='%s'][@id='%s::%s']", target, target, name);
mapTree = pkgSetTree->queryPropTree(xpath);
}
if (!mapTree)
{
xpath.clear().appendf("PackageMap[@querySet='%s'][@id='%s']", target, name);
mapTree = pkgSetTree->queryPropTree(xpath);
}
if (!mapTree)
throw MakeStringException(PKG_ACTIVATE_NOT_FOUND, "PackageMap %s not found on target %s", name, target);
makePackageActive(pkgSetTree, mapTree, target, activate);
}
}
IXRefNode * CXRefNodeManager::getXRefNode(const char* NodeName)
{
//DBGLOG("Node Name %s:",NodeName);
StringBuffer xpath;
xpath.appendf("Cluster[@name=\"%s\"]", NodeName);
//if not exists, add DFU/XREF/ClusterName branch to SDS
//not linked
Owned<IRemoteConnection> conn = querySDS().connect("/DFU/XREF",myProcessSession(),RTM_CREATE_QUERY|RTM_NONE ,INFINITE);
IPropertyTree* cluster_ptree = conn->queryRoot()->queryPropTree(xpath.str());
conn->commit();
if (cluster_ptree==0)
{
DBGLOG("Cluster[@name=%s] can't be found under /DFU/XREF", NodeName);
return 0 ;
}
return new CXRefNode(NodeName,conn);
}
IPropertyTree *getSwapNodeInfo(bool create)
{
Owned<IRemoteConnection> conn = querySDS().connect("/SwapNode", myProcessSession(), RTM_LOCK_WRITE|(create?RTM_CREATE_QUERY:0), 1000*60*5);
if (!conn) {
ERRLOG("SWAPNODE: could not connect to /SwapNode branch");
return NULL;
}
StringBuffer xpath;
xpath.appendf("Thor[@group=\"%s\"]",groupName.get());
Owned<IPropertyTree> info = conn->queryRoot()->getPropTree(xpath.str());
if (!info) {
if (!create) {
PROGLOG("SWAPNODE: no information for group %s",groupName.get());
return NULL;
}
info.set(conn->queryRoot()->addPropTree("Thor",createPTree("Thor")));
info->setProp("@group",groupName.get());
}
return info.getClear();
}
void doStuff()
{
Owned<IRemoteConnection> conn = querySDS().connect("/Orbit", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, DALI_TIMEOUT);
IPropertyTree *root = conn->queryRoot();
StringBuffer s;
if (root->getProp("TestBranch1",s)) {
printf("TestBranch1: read %s\n",s.str());
}
else {
// save as string
printf("TestBranch1: set (as string)\n",s.str());
root->setProp("TestBranch1",MyTestXML);
}
MemoryBuffer m;
if (root->getPropBin("TestBranch2",m)) {
m.append((byte)0); // add a NULL to returned data
const char *str = m.toByteArray();
printf("TestBranch2: read %s\n",str);
}
else {
// save as raw binary
printf("TestBranch2: set (as blob)\n",s.str());
root->setPropBin("TestBranch2",strlen(MyTestXML),MyTestXML); // include NULL
}
IPropertyTree *br3 = root->queryPropTree("TestBranch3");
if (br3) {
printf("read TestBranch3 as tree\n");
printf("Hello = %s\n",br3->queryProp("Hello"));
int n = br3->getPropInt("Bye/@num"); // update
printf("Bye num = %d\n",n);
br3->setPropInt("Bye/@num",n+1);
}
else {
// save as tree
printf("TestBranch3: set (as tree)\n",s.str());
br3 = createPTreeFromXMLString(MyTestXML); // parses and creates object tree
root->setPropTree("TestBranch3", br3);
}
}
static void loadMachineMap()
{
if (machinelist.ordinality())
return;
Owned<IRemoteConnection> conn = querySDS().connect("/Environment/Hardware", myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
if (!conn)
return;
IPropertyTree* root = conn->queryRoot();
Owned<IPropertyTreeIterator> machines= root->getElements("Computer");
if (machines->first()) {
do {
IPropertyTree &machine = machines->query();
SocketEndpoint ep(machine.queryProp("@netAddress"));
ep.port = getDaliServixPort();
const char *name = machine.queryProp("@name");
const char *state=machine.queryProp("@state");
CMachineEntry *entry = new CMachineEntry(name,ep,!state||stricmp(state,"Available")==0);
machinemap.setValue(name, entry);
machinelist.append(*entry);
} while (machines->next());
}
}
static void test1()
{
printf("Test SDS read/write\n");
Owned<IPropertyTree> ref = createPTree("DAREGRESS");
fn(1,2,3,0,ref);
StringBuffer refstr;
toXML(ref,refstr,0,XML_SortTags|XML_Format);
printf("Created reference size %d\n",refstr.length());
Owned<IRemoteConnection> conn = querySDS().connect("/DAREGRESS",myProcessSession(), RTM_CREATE, 1000000);
Rconn = conn;
IPropertyTree *root = conn->queryRoot();
fn(1,2,3,0,root);
conn.clear();
printf("Created test branch 1\n");
conn.setown(querySDS().connect("/DAREGRESS",myProcessSession(), RTM_DELETE_ON_DISCONNECT, 1000000));
root = conn->queryRoot();
StringBuffer s;
toXML(root,s,0,XML_SortTags|XML_Format);
if (strcmp(s.str(),refstr.str())!=0) {
ERROR("Branch 1 does not match");
}
else
printf("Branch 1 matches\n");
conn.clear();
conn.setown(querySDS().connect("/DAREGRESS",myProcessSession(), 0, 1000000));
if (conn)
ERROR("RTM_DELETE_ON_DISCONNECT failed");
Rconn = querySDS().connect("/DAREGRESS",myProcessSession(), RTM_CREATE, 1000000);
StringBuffer pn;
fn2(1,2,3,0,pn);
::Release(Rconn);
printf("Created test branch 2\n");
Rconn = NULL;
conn.setown(querySDS().connect("/DAREGRESS",myProcessSession(), RTM_DELETE_ON_DISCONNECT, 1000000));
root = conn->queryRoot();
toXML(root,s.clear(),0,XML_SortTags|XML_Format);
if (strcmp(s.str(),refstr.str())!=0) {
ERROR("Branch 2 does not match");
}
else
printf("Branch 2 matches\n");
conn.clear();
conn.setown(querySDS().connect("/DAREGRESS",myProcessSession(), 0, 1000000));
if (conn)
ERROR("RTM_DELETE_ON_DISCONNECT failed");
}
int main(int argc, const char *argv[])
{
InitModuleObjects();
int ret = 0;
bool dryRun = false;
bool offline = false;
StringAttr daliServer, envPath;
enum CmdType { cmd_none, cmd_swap, cmd_auto, cmd_history, cmd_email, cmd_swapped, cmd_reset, cmd_resetspares, cmd_addspares, cmd_removespares, cmd_resethistory };
CmdType cmd = cmd_none;
ArgvIterator iter(argc, argv);
try
{
bool stop=false;
StringArray params;
for (; !ret&&!iter.done(); iter.next())
{
const char *arg = iter.query();
if ('-' == *arg)
{
bool value;
if (iter.matchFlag(value, "-dryrun"))
dryRun = value;
else if (iter.matchFlag(value, "-offline"))
offline = value;
else
{
PROGLOG("Unknown option");
ret = 2;
usage();
break;
}
}
else
{
switch (cmd)
{
case cmd_none:
if (strieq("swap", arg))
cmd = cmd_swap;
else if (strieq("auto", arg))
cmd = cmd_auto;
else if (strieq("history", arg))
cmd = cmd_history;
else if (strieq("email", arg))
cmd = cmd_email;
else if (strieq("swapped", arg))
cmd = cmd_swapped;
else if (strieq("reset", arg))
cmd = cmd_reset;
else if (strieq("resetspares", arg))
cmd = cmd_resetspares;
else if (strieq("addspares", arg))
cmd = cmd_addspares;
else if (strieq("removespares", arg))
cmd = cmd_removespares;
else if (strieq("resethistory", arg))
cmd = cmd_resethistory;
else
{
PROGLOG("Unknown command");
usage();
ret = 2;
}
break;
default:
params.append(iter.query());
break;
}
}
}
unsigned requiredParams=UINT_MAX;
switch (cmd)
{
case cmd_swap:
requiredParams = 4;
break;
case cmd_addspares:
case cmd_removespares:
requiredParams = 3;
break;
case cmd_auto:
case cmd_history:
case cmd_email:
case cmd_swapped:
case cmd_reset:
case cmd_resetspares:
case cmd_resethistory:
requiredParams = 2;
break;
}
if (params.ordinality() < requiredParams)
{
usage();
ret = 2;
}
else
{
StringAttr daliServer = params.item(0);
StringAttr clusterName = params.item(1);
DaliClient dclient(daliServer);
StringBuffer logname;
splitFilename(argv[0], NULL, NULL, &logname, NULL);
addFileTimestamp(logname, true);
logname.append(".log");
StringBuffer lf;
openLogFile(lf, logname.str(),0,false,true);
queryStderrLogMsgHandler()->setMessageFields(MSGFIELD_prefix);
Owned<IRemoteConnection> conn = querySDS().connect("/Environment", myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
IPropertyTree *environment = conn->queryRoot();
StringBuffer xpath("Software/ThorCluster[@name=\"");
xpath.append(clusterName).append("\"]");
IPropertyTree *cluster = environment->queryPropTree(xpath.str());
if (!cluster)
{
PROGLOG("Unknown cluster: %s", clusterName.get());
ret = 3;
}
if (!ret)
{
Owned<IPropertyTree> options = createPTreeFromIPT(cluster);
conn.clear();
if (options&&options->getPropBool("@enableSysLog",true))
UseSysLogForOperatorMessages();
switch (cmd)
{
case cmd_auto:
{
if (!autoSwapNode(clusterName, dryRun))
ret = 3;
break;
}
case cmd_swap:
{
const char *oldip=params.item(2);
const char *newip=params.item(3);
if (!swapNode(clusterName, oldip, newip))
ret = 3;
break;
}
case cmd_history:
case cmd_swapped:
case cmd_email:
{
unsigned days = params.isItem(2) ? atoi(params.item(2)) : 0; // for history or swapped
switch (cmd)
{
case cmd_history:
swapNodeHistory(clusterName, days, NULL);
break;
case cmd_swapped:
swappedList(clusterName, days, NULL);
break;
case cmd_email:
{
bool sendSwapped = false;
bool sendHistory = false;
if (params.isItem(2))
{
if (strieq("swapped", params.item(2)))
sendSwapped = true;
else if (strieq("history", params.item(2)))
sendHistory = true;
}
emailSwap(clusterName, NULL, true, sendSwapped, sendHistory);
break;
}
}
break;
}
case cmd_reset:
case cmd_resetspares:
{
StringBuffer response;
if (!resetClusterGroup(clusterName, "ThorCluster", cmd==cmd_resetspares, response))
{
WARNLOG("%s", response.str());
ret = 3;
}
break;
}
case cmd_addspares:
case cmd_removespares:
{
SocketEndpointArray allEps;
unsigned p=2;
do
{
const char *ipOrRange = params.item(p);
SocketEndpointArray epa;
epa.fromText(ipOrRange, 0);
ForEachItemIn(e, epa)
allEps.append(epa.item(e));
p++;
}
while (p<params.ordinality());
StringBuffer response;
bool res;
if (cmd == cmd_addspares)
res = addClusterSpares(clusterName, "ThorCluster", allEps, response);
else
res = removeClusterSpares(clusterName, "ThorCluster", allEps, response);
if (!res)
{
WARNLOG("%s", response.str());
ret = 3;
}
break;
}
case cmd_resethistory:
{
Owned<IRemoteConnection> conn = querySDS().connect("/SwapNode", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
if (conn)
{
StringBuffer groupName;
getClusterGroupName(*options, groupName);
VStringBuffer xpath("Thor[@group=\"%s\"]", groupName.str());
if (conn->queryRoot()->removeProp(xpath.str()))
PROGLOG("SwapNode info for cluster %s removed", clusterName.get());
else
PROGLOG("SwapNode info for cluster %s not found", clusterName.get());
}
break;
}
}
}
}
UseSysLogForOperatorMessages(false);
}
catch (IException *e) {
EXCLOG(e,"SWAPNODE");
e->Release();
ret = -1;
}
ExitModuleObjects();
return ret;
}
void addPackageMapInfo(const char *xml, StringArray &filesNotFound, const char *process, const char *target, const char *pmid, const char *packageSetName, const char *lookupDaliIp, const char *srcCluster, const char *prefix, bool activate, bool overWrite, IUserDescriptor* userdesc, bool allowForeignFiles, bool preloadAll)
{
if (!xml || !*xml)
throw MakeStringExceptionDirect(PKG_INFO_NOT_DEFINED, "PackageMap info not provided");
if (srcCluster && *srcCluster)
{
if (!isProcessCluster(lookupDaliIp, srcCluster))
throw MakeStringException(PKG_INVALID_CLUSTER_TYPE, "Process cluster %s not found on %s DALI", srcCluster, lookupDaliIp ? lookupDaliIp : "local");
}
Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(target);
if (!clusterInfo)
throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Could not find information about target cluster %s ", target);
Owned<IPropertyTree> pmTree = createPTreeFromXMLString(xml);
if (!pmTree)
throw MakeStringExceptionDirect(PKG_INFO_NOT_DEFINED, "Invalid PackageMap info");
StringBuffer lcPmid(pmid);
pmid = lcPmid.toLowerCase().str();
pmTree->setProp("@id", pmid);
Owned<IPropertyTreeIterator> iter = pmTree->getElements("Package");
ForEach(*iter)
{
IPropertyTree &item = iter->query();
if (preloadAll)
item.setPropBool("@preload", true);
Owned<IPropertyTreeIterator> superFiles = item.getElements("SuperFile");
ForEach(*superFiles)
{
IPropertyTree &superFile = superFiles->query();
StringBuffer lc(superFile.queryProp("@id"));
const char *id = lc.toLowerCase().str();
if (*id == '~')
id++;
superFile.setProp("@id", id);
Owned<IPropertyTreeIterator> subFiles = superFile.getElements("SubFile");
ForEach(*subFiles)
{
IPropertyTree &subFile = subFiles->query();
id = subFile.queryProp("@value");
if (id && *id == '~')
{
StringAttr value(id+1);
subFile.setProp("@value", value.get());
}
}
}
}
VStringBuffer xpath("PackageMap[@id='%s']", pmid);
Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageMaps", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
IPropertyTree *packageMaps = globalLock->queryRoot();
IPropertyTree *pmExisting = packageMaps->queryPropTree(xpath);
xpath.appendf("[@querySet='%s']", target);
Owned<IPropertyTree> pkgSet = getPkgSetRegistry(process, false);
IPropertyTree *psEntry = pkgSet->queryPropTree(xpath);
if (!overWrite && (psEntry || pmExisting))
throw MakeStringException(PKG_NAME_EXISTS, "Package name %s already exists, either delete it or specify overwrite", pmid);
cloneFileInfoToDali(filesNotFound, pmTree, lookupDaliIp, clusterInfo, srcCluster, prefix, overWrite, userdesc, allowForeignFiles);
if (pmExisting)
packageMaps->removeTree(pmExisting);
packageMaps->addPropTree("PackageMap", pmTree.getClear());
if (!psEntry)
{
psEntry = pkgSet->addPropTree("PackageMap", createPTree("PackageMap"));
psEntry->setProp("@id", pmid);
psEntry->setProp("@querySet", target);
}
makePackageActive(pkgSet, psEntry, target, activate);
}
unsigned applyNodes(const char *grpip, ApplyMode mode, unsigned ver, bool isdali, bool quiet)
{
SocketEndpointArray eps;
if (isdali&&(stricmp(grpip,"all")==0)) {
Owned<IRemoteConnection> conn = querySDS().connect("/Environment/Software", myProcessSession(), RTM_LOCK_READ, SDS_CONNECT_TIMEOUT);
if (!conn)
return 0;
IPropertyTree* root = conn->queryRoot();
Owned<IPropertyTreeIterator> clusters= root->getElements("ThorCluster");
unsigned ret = 0;
if (clusters->first()) {
do {
IPropertyTree &cluster = clusters->query();
ret += applyNodes(cluster.queryProp("@name"),mode,ver,true,quiet);
} while (clusters->next());
}
return ret;
}
SocketEndpointArray result;
StringAttrArray resultstr;
if (!isdali||!getCluster(grpip,eps)) {
SocketEndpoint ep(grpip);
if (ep.isNull()) {
ERRLOG("%s is not a group name or ip",grpip);
return 0;
}
if (ep.port==0)
ep.port = getDaliServixPort();
eps.append(ep);
}
PointerIArrayOf<ISocket> sockets;
unsigned to=10*1000;
unsigned n=eps.ordinality(); // use approx log scale (timeout is long but only for failure situation)
while (n>1) {
n/=2;
to+=10*1000;
}
if (!quiet&&(n>1))
PROGLOG("Scanning %s...",grpip);
multiConnect(eps,sockets,to);
CriticalSection sect;
class casyncfor: public CAsyncFor
{
SocketEndpointArray &eps;
PointerIArrayOf<ISocket> &sockets;
ApplyMode mode;
unsigned ver;
SocketEndpointArray &result;
StringAttrArray &resultstr;
CriticalSection §
public:
casyncfor(ApplyMode _mode, unsigned _ver,SocketEndpointArray &_eps,PointerIArrayOf<ISocket> &_sockets,SocketEndpointArray &_result, StringAttrArray &_resultstr,CriticalSection &_sect)
: eps(_eps), sockets(_sockets), result(_result), resultstr(_resultstr), sect(_sect)
{
mode = _mode;
ver = _ver;
}
void Do(unsigned i)
{
ISocket *sock = sockets.item(i);
StringBuffer epstr;
SocketEndpoint ep = eps.item(i);
ep.getUrlStr(epstr);
// PROGLOG("T.1 %s %x",epstr.str(),(unsigned)sock);
StringBuffer verstr;
unsigned rver=0;
if (sock) {
rver = getRemoteVersion(sock, verstr);
switch (mode) {
case AMcheck:
if (rver!=0)
return;
case AMver: {
CriticalBlock block(sect);
result.append(ep);
StringBuffer ln;
ln.append(rver).append(",\"").append(verstr).append('"');
resultstr.append(* new StringAttrItem(ln.str()));
}
return;
case AMstopver:
case AMcheckver:
case AMcheckvermajor: {
// compares versions up to the '-'
const char *rv = verstr.str();
const char *v = remoteServerVersionString();
if (mode!=AMcheckvermajor) {
while (*v&&(*v!='-')&&(*v==*rv)) {
v++;
rv++;
}
}
if ((*rv==*v)&&(rver==ver))
return;
while (*rv&&(*rv!='-'))
rv++;
verstr.setLength(rv-verstr.str());
if ((mode==AMcheckver)||(mode==AMcheckvermajor))
break;
}
// fall through
case AMstop:
{
unsigned err = stopRemoteServer(sock);
if (err!=0) {
ERRLOG("Could not stop server on %s, %d returned",epstr.str(),err);
if (mode!=AMstopver)
return; // even though failed to stop - still return code
}
else
Sleep(1000); // let stop
}
break;
default:
return;
}
}
CriticalBlock block(sect);
result.append(ep);
if ((mode!=AMver)&&(mode!=AMcheckver)&&(mode!=AMcheckvermajor)&&(mode!=AMstopver))
resultstr.append(* new StringAttrItem(""));
else
resultstr.append(* new StringAttrItem(verstr.str()));
}
} afor(mode,ver,eps,sockets,result,resultstr,sect);
afor.For(eps.ordinality(), 10, false, true);
if (result.ordinality()==0)
return 0;
switch (mode) {
case AMstopver:
case AMcheckver:
case AMcheckvermajor:
if (!quiet) {
StringBuffer epstr;
ForEachItemIn(i,result) {
result.item(i).getUrlStr(epstr.clear());
StringAttrItem &attr = resultstr.item(i);
if (attr.text.length()==0)
ERRLOG("%s: %s not running DAFILESRV",grpip,epstr.str());
else
ERRLOG("%s: %s %s running DAFILESRV version %s",grpip,(mode==AMstopver)?"was":"is",epstr.str(),attr.text.get());
}
unsigned numok = eps.ordinality()-result.ordinality();
if (mode==AMcheckvermajor)
PROGLOG("%s: %d node%s running version %.1f of DAFILESRV",grpip,numok,(numok!=1)?"s":"",((double)FILESRV_VERSION)/10.0);
else {
StringBuffer vs;
const char *v = remoteServerVersionString();
while (*v&&(*v!='-'))
vs.append(*(v++));
PROGLOG("%s: %d node%s running version %s of DAFILESRV",grpip,numok,(numok!=1)?"s":"",vs.str());
}
}