// This test verifies that the environment secrets are resolved when launching a
// task.
TEST_F(EnvironmentSecretIsolatorTest, ResolveSecret)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
mesos::internal::slave::Flags flags = CreateSlaveFlags();
Fetcher fetcher(flags);
Try<SecretResolver*> secretResolver = SecretResolver::create();
EXPECT_SOME(secretResolver);
Try<MesosContainerizer*> containerizer =
MesosContainerizer::create(flags, false, &fetcher, secretResolver.get());
EXPECT_SOME(containerizer);
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave =
StartSlave(detector.get(), containerizer.get());
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<std::vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
EXPECT_FALSE(offers->empty());
const string commandString = strings::format(
"env; test \"$%s\" = \"%s\"",
SECRET_ENV_NAME,
SECRET_VALUE).get();
CommandInfo command;
command.set_value(commandString);
// Request a secret.
// TODO(kapil): Update createEnvironment() to support secrets.
mesos::Environment::Variable *env =
command.mutable_environment()->add_variables();
env->set_name(SECRET_ENV_NAME);
env->set_type(mesos::Environment::Variable::SECRET);
mesos::Secret* secret = env->mutable_secret();
secret->set_type(Secret::VALUE);
secret->mutable_value()->set_data(SECRET_VALUE);
TaskInfo task = createTask(
offers.get()[0].slave_id(),
Resources::parse("cpus:0.1;mem:32").get(),
command);
// NOTE: Successful tasks will output two status updates.
Future<TaskStatus> statusRunning;
Future<TaskStatus> statusFinished;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&statusRunning))
.WillOnce(FutureArg<1>(&statusFinished));
driver.launchTasks(offers.get()[0].id(), {task});
AWAIT_READY(statusRunning);
EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
AWAIT_READY(statusFinished);
EXPECT_EQ(TASK_FINISHED, statusFinished.get().state());
driver.stop();
driver.join();
}
unsigned applyNodes(const char *grpip, ApplyMode mode, unsigned ver, bool isdali, bool quiet)
{
SocketEndpointArray eps;
if (isdali&&(stricmp(grpip,"all")==0)) {
Owned<IRemoteConnection> conn = querySDS().connect("/Environment/Software", myProcessSession(), RTM_LOCK_READ, SDS_CONNECT_TIMEOUT);
if (!conn)
return 0;
IPropertyTree* root = conn->queryRoot();
Owned<IPropertyTreeIterator> clusters= root->getElements("ThorCluster");
unsigned ret = 0;
if (clusters->first()) {
do {
IPropertyTree &cluster = clusters->query();
ret += applyNodes(cluster.queryProp("@name"),mode,ver,true,quiet);
} while (clusters->next());
}
return ret;
}
SocketEndpointArray result;
StringAttrArray resultstr;
if (!isdali||!getCluster(grpip,eps)) {
SocketEndpoint ep(grpip);
if (ep.isNull()) {
ERRLOG("%s is not a group name or ip",grpip);
return 0;
}
if (ep.port==0)
ep.port = getDaliServixPort();
eps.append(ep);
}
PointerIArrayOf<ISocket> sockets;
unsigned to=10*1000;
unsigned n=eps.ordinality(); // use approx log scale (timeout is long but only for failure situation)
while (n>1) {
n/=2;
to+=10*1000;
}
if (!quiet&&(n>1))
PROGLOG("Scanning %s...",grpip);
multiConnect(eps,sockets,to);
CriticalSection sect;
class casyncfor: public CAsyncFor
{
SocketEndpointArray &eps;
PointerIArrayOf<ISocket> &sockets;
ApplyMode mode;
unsigned ver;
SocketEndpointArray &result;
StringAttrArray &resultstr;
CriticalSection §
public:
casyncfor(ApplyMode _mode, unsigned _ver,SocketEndpointArray &_eps,PointerIArrayOf<ISocket> &_sockets,SocketEndpointArray &_result, StringAttrArray &_resultstr,CriticalSection &_sect)
: eps(_eps), sockets(_sockets), result(_result), resultstr(_resultstr), sect(_sect)
{
mode = _mode;
ver = _ver;
}
void Do(unsigned i)
{
ISocket *sock = sockets.item(i);
StringBuffer epstr;
SocketEndpoint ep = eps.item(i);
ep.getUrlStr(epstr);
// PROGLOG("T.1 %s %x",epstr.str(),(unsigned)sock);
StringBuffer verstr;
unsigned rver=0;
if (sock) {
rver = getRemoteVersion(sock, verstr);
switch (mode) {
case AMcheck:
if (rver!=0)
return;
case AMver: {
CriticalBlock block(sect);
result.append(ep);
StringBuffer ln;
ln.append(rver).append(",\"").append(verstr).append('"');
resultstr.append(* new StringAttrItem(ln.str()));
}
return;
case AMstopver:
case AMcheckver:
case AMcheckvermajor: {
// compares versions up to the '-'
const char *rv = verstr.str();
const char *v = remoteServerVersionString();
if (mode!=AMcheckvermajor) {
while (*v&&(*v!='-')&&(*v==*rv)) {
v++;
rv++;
}
}
if ((*rv==*v)&&(rver==ver))
return;
while (*rv&&(*rv!='-'))
rv++;
verstr.setLength(rv-verstr.str());
if ((mode==AMcheckver)||(mode==AMcheckvermajor))
break;
}
// fall through
case AMstop:
{
unsigned err = stopRemoteServer(sock);
if (err!=0) {
ERRLOG("Could not stop server on %s, %d returned",epstr.str(),err);
if (mode!=AMstopver)
return; // even though failed to stop - still return code
}
else
Sleep(1000); // let stop
}
break;
default:
return;
}
}
CriticalBlock block(sect);
result.append(ep);
if ((mode!=AMver)&&(mode!=AMcheckver)&&(mode!=AMcheckvermajor)&&(mode!=AMstopver))
resultstr.append(* new StringAttrItem(""));
else
resultstr.append(* new StringAttrItem(verstr.str()));
}
} afor(mode,ver,eps,sockets,result,resultstr,sect);
afor.For(eps.ordinality(), 10, false, true);
if (result.ordinality()==0)
return 0;
switch (mode) {
case AMstopver:
case AMcheckver:
case AMcheckvermajor:
if (!quiet) {
StringBuffer epstr;
ForEachItemIn(i,result) {
result.item(i).getUrlStr(epstr.clear());
StringAttrItem &attr = resultstr.item(i);
if (attr.text.length()==0)
ERRLOG("%s: %s not running DAFILESRV",grpip,epstr.str());
else
ERRLOG("%s: %s %s running DAFILESRV version %s",grpip,(mode==AMstopver)?"was":"is",epstr.str(),attr.text.get());
}
unsigned numok = eps.ordinality()-result.ordinality();
if (mode==AMcheckvermajor)
PROGLOG("%s: %d node%s running version %.1f of DAFILESRV",grpip,numok,(numok!=1)?"s":"",((double)FILESRV_VERSION)/10.0);
else {
StringBuffer vs;
const char *v = remoteServerVersionString();
while (*v&&(*v!='-'))
vs.append(*(v++));
PROGLOG("%s: %d node%s running version %s of DAFILESRV",grpip,numok,(numok!=1)?"s":"",vs.str());
}
}
void sendRequest()
{
if (!loggingManager)
{
printf("No logging manager.\n");
return;
}
StringBuffer action, option, status;
testData->getProp("action", action);
testData->getProp("option", option);
if (action.length() && strieq(action.str(), "getTransactionSeed"))
{
StringBuffer transactionSeed;
loggingManager->getTransactionSeed(transactionSeed, status);
if (transactionSeed.length())
printf("Got transactionSeed: <%s>\n", transactionSeed.str());
}
else if (action.length() && strieq(action.str(), "UpdateLog"))
{
IPropertyTree* logContentTree = testData->queryPropTree("LogContent");
if (!logContentTree)
{
printf("can't read log content.\n");
return;
}
StringBuffer logContentXML;
toXML(logContentTree, logContentXML);
printf("log content: <%s>.\n", logContentXML.str());
Owned<IEspContext> espContext = createEspContext();
const char* userName = logContentTree->queryProp("ESPContext/UserName");
const char* sourceIP = logContentTree->queryProp("ESPContext/SourceIP");
short servPort = logContentTree->getPropInt("ESPContext/Port");
espContext->setUserID(userName);
espContext->setServAddress(sourceIP, servPort);
const char* backEndResp = logContentTree->queryProp("BackEndResponse");
IPropertyTree* userContextTree = logContentTree->queryPropTree("MyUserContext");
IPropertyTree* userRequestTree = logContentTree->queryPropTree("MyUserRequest");
IPropertyTree* userRespTree = logContentTree->queryPropTree("MyUserResponseEx");
IPropertyTree* logDatasetsTree = logContentTree->queryPropTree("LogDatasets");
StringBuffer userContextXML, userRequestXML, userRespXML, logDatasetsXML;
toXML(logDatasetsTree, logDatasetsXML);
toXML(userRespTree, userRespXML);
toXML(userContextTree, userContextXML);
toXML(userRequestTree, userRequestXML);
printf("userContextXML: <%s>.\n", userContextXML.str());
printf("userRequestXML: <%s>.\n", userRequestXML.str());
printf("userRespXML: <%s>.\n", userRespXML.str());
printf("backEndResp: <%s>.\n", backEndResp);
//Sleep(5000); //Waiting for loggingManager to start
loggingManager->updateLog(option.str(), *espContext, userContextTree, userRequestTree, backEndResp, userRespXML.str(), logDatasetsXML.str(), status);
}
else if (action.length() && strieq(action.str(), "UpdateLog1"))
{
IPropertyTree* logContentTree = testData->queryPropTree("LogContent");
if (!logContentTree)
{
printf("can't read log content.\n");
return;
}
StringBuffer logContentXML;
toXML(logContentTree, logContentXML);
printf("log content: <%s>.\n", logContentXML.str());
//Sleep(5000); //Waiting for loggingManager to start
loggingManager->updateLog(option.str(), logContentXML.str(), status);
}
else
printf("Invalid action.\n");
}
unsigned idleTime() { return server.get()?server->idleTime():0; }
TEST_P(MemoryIsolatorTest, ROOT_MemUsage)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
slave::Flags flags = CreateSlaveFlags();
flags.isolation = GetParam();
Fetcher fetcher(flags);
Try<MesosContainerizer*> _containerizer =
MesosContainerizer::create(flags, true, &fetcher);
ASSERT_SOME(_containerizer);
Owned<MesosContainerizer> containerizer(_containerizer.get());
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(
detector.get(),
containerizer.get());
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched,
DEFAULT_FRAMEWORK_INFO,
master.get()->pid,
DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
ASSERT_FALSE(offers->empty());
TaskInfo task = createTask(offers.get()[0], "sleep 120");
Future<TaskStatus> statusRunning;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&statusRunning));
driver.launchTasks(offers.get()[0].id(), {task});
AWAIT_READY(statusRunning);
EXPECT_EQ(TASK_RUNNING, statusRunning->state());
Future<hashset<ContainerID>> containers = containerizer->containers();
AWAIT_READY(containers);
ASSERT_EQ(1u, containers->size());
ContainerID containerId = *(containers->begin());
Future<ResourceStatistics> usage = containerizer->usage(containerId);
AWAIT_READY(usage);
// TODO(jieyu): Consider using a program that predictably increases
// RSS so that we can set more meaningful expectation here.
EXPECT_LT(0u, usage->mem_rss_bytes());
driver.stop();
driver.join();
}
int CDfuPlusHelper::despray()
{
const char* srcname = globals->queryProp("srcname");
if(srcname == NULL)
throw MakeStringException(-1, "srcname not specified");
const char* dstxml = globals->queryProp("dstxml");
const char* dstip = globals->queryProp("dstip");
const char* dstfile = globals->queryProp("dstfile");
bool nowait = globals->getPropBool("nowait", false);
MemoryBuffer xmlbuf;
if(dstxml == NULL)
{
if(dstfile == NULL)
throw MakeStringException(-1, "dstfile not specified");
if(dstip == NULL) {
#ifdef DAFILESRV_LOCAL
progress("dstip not specified - assuming spray from local machine\n");
dstip = ".";
#else
throw MakeStringException(-1, "dstip not specified");
#endif
}
}
else
{
if(dstip != NULL || dstfile != NULL)
throw MakeStringException(-1, "dstip/dstfile and dstxml can't be used at the same time");
StringBuffer buf;
buf.loadFile(dstxml);
int len = buf.length();
xmlbuf.setBuffer(len, buf.detach(), true);
}
if(dstxml == NULL)
info("\nDespraying %s to host %s file %s\n", srcname, dstip, dstfile);
else
info("\nDespraying %s\n", srcname);
Owned<IClientDespray> req = sprayclient->createDesprayRequest();
req->setSourceLogicalName(srcname);
if(dstxml == NULL)
{
req->setDestIP(dstip);
req->setDestPath(dstfile);
}
else
req->setDstxml(xmlbuf);
req->setOverwrite(globals->getPropBool("overwrite", false));
if(globals->hasProp("connect"))
req->setMaxConnections(globals->getPropInt("connect"));
if(globals->hasProp("throttle"))
req->setThrottle(globals->getPropInt("throttle"));
if(globals->hasProp("transferbuffersize"))
req->setTransferBufferSize(globals->getPropInt("transferbuffersize"));
if(globals->hasProp("norecover"))
req->setNorecover(globals->getPropBool("norecover", false));
else if(globals->hasProp("noRecover"))
req->setNorecover(globals->getPropBool("noRecover", false));
if(globals->hasProp("splitprefix"))
req->setSplitprefix(globals->queryProp("splitprefix"));
else if(globals->hasProp("splitPrefix"))
req->setSplitprefix(globals->queryProp("splitPrefix"));
if(globals->hasProp("wrap"))
req->setWrap(globals->getPropBool("wrap", false));
if(globals->hasProp("multicopy"))
req->setMultiCopy(globals->getPropBool("multicopy", false));
else if(globals->hasProp("multiCopy"))
req->setMultiCopy(globals->getPropBool("multiCopy", false));
if(globals->hasProp("compress"))
req->setCompress(globals->getPropBool("compress", false));
if(globals->hasProp("encrypt"))
req->setEncrypt(globals->queryProp("encrypt"));
if(globals->hasProp("decrypt"))
req->setDecrypt(globals->queryProp("decrypt"));
SocketEndpoint localep;
StringBuffer localeps;
if (checkLocalDaFileSvr(dstip,localep))
dstip = localep.getUrlStr(localeps).str();
Owned<IClientDesprayResponse> result = sprayclient->Despray(req);
const char* wuid = result->getWuid();
if(wuid == NULL || *wuid == '\0')
exc(result->getExceptions(),"despraying");
else
{
const char* jobname = globals->queryProp("jobname");
if(jobname && *jobname)
updatejobname(wuid, jobname);
info("Submitted WUID %s\n", wuid);
if(!nowait)
waitToFinish(wuid);
}
return 0;
}
int CDfuPlusHelper::copysuper()
{
const char* srcname = globals->queryProp("srcname");
if(srcname == NULL)
throw MakeStringException(-1, "srcname not specified");
const char* dstname = globals->queryProp("dstname");
if(dstname == NULL)
throw MakeStringException(-1, "dstname not specified");
const char* dstcluster = globals->queryProp("dstcluster");
if(dstcluster == NULL)
throw MakeStringException(-1, "dstcluster not specified");
const char* srcdali = globals->queryProp("srcdali");
if(srcdali == NULL)
throw MakeStringException(-1, "srcdali not specified");
const char* dstclusterroxie = globals->queryProp("dstclusterroxie"); // not sure if this applicable
const char* srcusername = globals->queryProp("srcusername");
const char* srcpassword = globals->queryProp("srcpassword");
bool nowait = globals->getPropBool("nowait", false);
info("\nCopying superfile from %s to %s\n", srcname, dstname);
Owned<IClientCopy> req = sprayclient->createCopyRequest();
req->setSuperCopy(true);
req->setSourceLogicalName(srcname);
req->setDestLogicalName(dstname);
req->setDestGroup(dstcluster);
if(dstclusterroxie && !stricmp(dstclusterroxie, "Yes")) // not sure if this applicable
req->setDestGroupRoxie("Yes");
req->setSourceDali(srcdali);
if(srcusername && *srcusername)
req->setSrcusername(srcusername);
if(srcpassword && *srcpassword)
req->setSrcpassword(srcpassword);
req->setOverwrite(globals->getPropBool("overwrite", false));
/* the foolowing are not currently supported serverside */
req->setReplicate(globals->getPropBool("replicate", true));
req->setReplicateOffset(globals->getPropInt("replicateoffset",1));
if(globals->hasProp("nosplit"))
req->setNosplit(globals->getPropBool("nosplit", false));
if(globals->hasProp("push")) {
if (globals->getPropBool("push"))
req->setPush(true);
else
req->setPull(true);
}
if(globals->hasProp("compress"))
req->setCompress(globals->getPropBool("compress", false));
if(globals->hasProp("encrypt"))
req->setEncrypt(globals->queryProp("encrypt"));
if(globals->hasProp("decrypt"))
req->setDecrypt(globals->queryProp("decrypt"));
if(globals->hasProp("connect"))
req->setMaxConnections(globals->getPropInt("connect"));
if(globals->hasProp("throttle"))
req->setThrottle(globals->getPropInt("throttle"));
if(globals->hasProp("transferbuffersize"))
req->setTransferBufferSize(globals->getPropInt("transferbuffersize"));
if(globals->hasProp("wrap"))
req->setWrap(globals->getPropBool("wrap", false));
if(globals->hasProp("norecover"))
req->setNorecover(globals->getPropBool("norecover", false));
else if(globals->hasProp("noRecover"))
req->setNorecover(globals->getPropBool("noRecover", false));
Owned<IClientCopyResponse> result = sprayclient->Copy(req);
const char* ret = result->getResult();
if(ret == NULL || *ret == '\0')
exc(result->getExceptions(),"copying");
else if (stricmp(ret,"OK")==0)
info("Superfile copy completed\n");
else
info("Superfile copy failed: %s\n",ret);
return 0;
}
void kill()
{
CMasterActivity::kill();
originalIndexFile.clear();
newIndexFile.clear();
}
void init()
{
helper = (IHThorKeyDiffArg *)queryHelper();
originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), helper->queryOriginalName()));
newIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), helper->queryUpdatedName()));
if (originalIndexFile->numParts() != newIndexFile->numParts())
throw MakeActivityException(this, TE_KeyDiffIndexSizeMismatch, "Index %s and %s differ in width", helper->queryOriginalName(), helper->queryUpdatedName());
if (originalIndexFile->querySuperFile() || newIndexFile->querySuperFile())
throw MakeActivityException(this, 0, "Diffing super files not supported");
width = originalIndexFile->numParts();
originalDesc.setown(originalIndexFile->getFileDescriptor());
newIndexDesc.setown(newIndexFile->getFileDescriptor());
Owned<IPartDescriptor> tlkDesc = originalDesc->getPart(originalDesc->numParts()-1);
const char *kind = tlkDesc->queryProperties().queryProp("@kind");
local = NULL == kind || 0 != stricmp("topLevelKey", kind);
if (!local)
width--; // 1 part == No n distributed / Monolithic key
if (width > container.queryJob().querySlaves())
throw MakeActivityException(this, 0, "Unsupported: keydiff(%s, %s) - Cannot diff a key that's wider(%d) than the target cluster size(%d)", originalIndexFile->queryLogicalName(), newIndexFile->queryLogicalName(), width, container.queryJob().querySlaves());
queryThorFileManager().noteFileRead(container.queryJob(), originalIndexFile);
queryThorFileManager().noteFileRead(container.queryJob(), newIndexFile);
IArrayOf<IGroup> groups;
fillClusterArray(container.queryJob(), helper->queryOutputName(), clusters, groups);
patchDesc.setown(queryThorFileManager().create(container.queryJob(), helper->queryOutputName(), clusters, groups, 0 != (KDPoverwrite & helper->getFlags()), 0, !local, width));
}
// This test ensures that when explicit acknowledgements are enabled,
// acknowledgements for master-generated updates are dropped by the
// driver. We test this by creating an invalid task that uses no
// resources.
TEST_F(MesosSchedulerDriverTest, ExplicitAcknowledgementsMasterGeneratedUpdate)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched,
DEFAULT_FRAMEWORK_INFO,
master.get()->pid,
false,
DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
// Ensure no status update acknowledgements are sent to the master.
EXPECT_NO_FUTURE_CALLS(
mesos::scheduler::Call(),
mesos::scheduler::Call::ACKNOWLEDGE,
_ ,
master.get()->pid);
driver.start();
AWAIT_READY(offers);
EXPECT_NE(0u, offers->size());
// Launch a task using no resources.
TaskInfo task;
task.set_name("");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id());
task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO);
vector<TaskInfo> tasks;
tasks.push_back(task);
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status));
driver.launchTasks(offers.get()[0].id(), tasks);
AWAIT_READY(status);
ASSERT_EQ(TASK_ERROR, status->state());
ASSERT_EQ(TaskStatus::SOURCE_MASTER, status->source());
ASSERT_EQ(TaskStatus::REASON_TASK_INVALID, status->reason());
// Now send the acknowledgement.
driver.acknowledgeStatusUpdate(status.get());
// Settle the clock to ensure driver processes the acknowledgement,
// which should get dropped due to having come from the master.
Clock::pause();
Clock::settle();
driver.stop();
driver.join();
}
void done()
{
StringBuffer scopedName;
queryThorFileManager().addScope(container.queryJob(), helper->queryOutputName(), scopedName);
Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock();
Owned<IWUResult> r = wu->updateResultBySequence(helper->getSequence());
r->setResultStatus(ResultStatusCalculated);
r->setResultLogicalName(scopedName.str());
r.clear();
wu.clear();
IPropertyTree &patchProps = patchDesc->queryProperties();
setExpiryTime(patchProps, helper->getExpiryDays());
IPropertyTree &originalProps = originalDesc->queryProperties();;
if (originalProps.queryProp("ECL"))
patchProps.setProp("ECL", originalProps.queryProp("ECL"));
if (originalProps.getPropBool("@local"))
patchProps.setPropBool("@local", true);
container.queryTempHandler()->registerFile(helper->queryOutputName(), container.queryOwner().queryGraphId(), 0, false, WUFileStandard, &clusters);
Owned<IDistributedFile> patchFile;
// set part sizes etc
queryThorFileManager().publish(container.queryJob(), helper->queryOutputName(), false, *patchDesc, &patchFile, 0, false);
try { // set file size
if (patchFile) {
__int64 fs = patchFile->getFileSize(true,false);
if (fs!=-1)
patchFile->queryAttributes().setPropInt64("@size",fs);
}
}
catch (IException *e) {
EXCLOG(e,"keydiff setting file size");
e->Release();
}
// Add a new 'Patch' description to the secondary key.
DistributedFilePropertyLock lock(newIndexFile);
IPropertyTree &fileProps = lock.queryAttributes();
StringBuffer path("Patch[@name=\"");
path.append(scopedName.str()).append("\"]");
IPropertyTree *patch = fileProps.queryPropTree(path.str());
if (!patch) patch = fileProps.addPropTree("Patch", createPTree());
patch->setProp("@name", scopedName.str());
unsigned checkSum;
if (patchFile->getFileCheckSum(checkSum))
patch->setPropInt64("@checkSum", checkSum);
IPropertyTree *index = patch->setPropTree("Index", createPTree());
index->setProp("@name", originalIndexFile->queryLogicalName());
if (originalIndexFile->getFileCheckSum(checkSum))
index->setPropInt64("@checkSum", checkSum);
}
// Ensures that when a scheduler enables explicit acknowledgements
// on the driver, there are no implicit acknowledgements sent, and
// the call to 'acknowledgeStatusUpdate' sends the ack to the master.
TEST_F(MesosSchedulerDriverTest, ExplicitAcknowledgements)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
TestContainerizer containerizer(&exec);
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched,
DEFAULT_FRAMEWORK_INFO,
master.get()->pid,
false,
DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(LaunchTasks(DEFAULT_EXECUTOR_INFO, 1, 1, 16, "*"))
.WillRepeatedly(Return()); // Ignore subsequent offers.
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status));
// Ensure no status update acknowledgements are sent from the driver
// to the master until the explicit acknowledgement is sent.
EXPECT_NO_FUTURE_CALLS(
mesos::scheduler::Call(),
mesos::scheduler::Call::ACKNOWLEDGE,
_ ,
master.get()->pid);
EXPECT_CALL(exec, registered(_, _, _, _));
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
driver.start();
AWAIT_READY(status);
// Settle the clock to ensure driver finishes processing the status
// update, we want to ensure that no implicit acknowledgement gets
// sent.
Clock::pause();
Clock::settle();
// Now send the acknowledgement.
Future<mesos::scheduler::Call> acknowledgement = FUTURE_CALL(
mesos::scheduler::Call(),
mesos::scheduler::Call::ACKNOWLEDGE,
_,
master.get()->pid);
driver.acknowledgeStatusUpdate(status.get());
AWAIT_READY(acknowledgement);
driver.stop();
driver.join();
}
void CDiskWriteSlaveActivityBase::open()
{
if (dlfn.isExternal() && !firstNode())
{
input.setown(createDataLinkSmartBuffer(this, inputs.item(0), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, false, &container.queryJob().queryIDiskUsage()));
startInput(input);
if (!rfsQueryParallel)
{
ActPrintLog("Blocked, waiting for previous part to complete write");
CMessageBuffer msg;
if (!receiveMsg(msg, container.queryJob().queryMyRank()-1, mpTag))
return;
rowcount_t prevRows;
msg.read(prevRows);
msg.read(tempExternalName); // reuse temp filename, last node will rename
ActPrintLog("Previous write row count = %"RCPF"d", prevRows);
}
}
else
{
input.set(inputs.item(0));
startInput(input);
}
processed = THORDATALINK_STARTED;
bool extend = 0 != (diskHelperBase->getFlags() & TDWextend);
if (extend)
ActPrintLog("Extending file %s", fName.get());
size32_t exclsz = 0;
calcFileCrc = true;
bool external = dlfn.isExternal();
bool query = dlfn.isQuery();
if (query && compress)
UNIMPLEMENTED;
bool direct = query || (external && !firstNode());
bool rename = !external || (!query && lastNode());
Owned<IFileIO> iFileIO = createMultipleWrite(this, *partDesc, exclsz, compress, extend||(external&&!query), ecomp, this, direct, rename, &abortSoon, (external&&!query) ? &tempExternalName : NULL);
if (compress)
{
ActPrintLog("Performing row compression on output file: %s", fName.get());
calcFileCrc = false;
}
Owned<IFileIOStream> stream;
if (wantRaw())
{
outraw.setown(createBufferedIOStream(iFileIO));
stream.set(outraw);
}
else
{
stream.setown(createIOStream(iFileIO));
out.setown(createRowWriter(stream,::queryRowSerializer(input),::queryRowAllocator(input),grouped,calcFileCrc,false)); // flushed by close
}
CDfsLogicalFileName dlfn;
dlfn.set(logicalFilename);
if (extend || (external && !query))
stream->seek(0,IFSend);
ActPrintLog("Created output stream for %s", fName.get());
}
foreachvalue (const Owned<Store>& store, stores) {
recovers.push_back(store->recover());
}
bool CDfuPlusHelper::fixedSpray(const char* srcxml,const char* srcip,const char* srcfile,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format, StringBuffer &retwuid, StringBuffer &except)
{
int recordsize;
if(stricmp(format, "recfmvb") == 0) {
recordsize = RECFMVB_RECSIZE_ESCAPE; // special value for recfmvb
}
else if(stricmp(format, "recfmv") == 0) {
recordsize = RECFMV_RECSIZE_ESCAPE; // special value for recfmv
}
else if(stricmp(format, "variable") == 0) {
recordsize = PREFIX_VARIABLE_RECSIZE_ESCAPE; // special value for variable
}
else if(stricmp(format, "variablebigendian") == 0) {
recordsize = PREFIX_VARIABLE_BIGENDIAN_RECSIZE_ESCAPE; // special value for variable bigendian
}
else {
const char* rsstr = globals->queryProp("recordsize");
recordsize = rsstr?atoi(rsstr):0;
if(!recordsize && !globals->hasProp("nosplit"))
throw MakeStringException(-1, "recordsize not specified for fixed");
}
Owned<IClientSprayFixed> req = sprayclient->createSprayFixedRequest();
if(srcxml == NULL)
{
req->setSourceIP(srcip);
req->setSourcePath(srcfile);
}
else
req->setSrcxml(xmlbuf);
if(recordsize != 0)
req->setSourceRecordSize(recordsize);
if(dstcluster != NULL)
req->setDestGroup(dstcluster);
req->setDestLogicalName(dstname);
req->setOverwrite(globals->getPropBool("overwrite", false));
req->setReplicate(globals->getPropBool("replicate", true));
req->setReplicateOffset(globals->getPropInt("replicateoffset",1));
if(globals->hasProp("prefix"))
req->setPrefix(globals->queryProp("prefix"));
if(globals->hasProp("nosplit"))
req->setNosplit(globals->getPropBool("nosplit", false));
if(globals->hasProp("wrap"))
req->setWrap(globals->getPropBool("wrap", false));
if(globals->hasProp("compress"))
req->setCompress(globals->getPropBool("compress", false));
if(globals->hasProp("encrypt"))
req->setEncrypt(globals->queryProp("encrypt"));
if(globals->hasProp("decrypt"))
req->setDecrypt(globals->queryProp("decrypt"));
if(globals->hasProp("push")) {
if (globals->getPropBool("push"))
req->setPush(true);
else
req->setPull(true);
}
if(globals->hasProp("norecover"))
req->setNorecover(globals->getPropBool("norecover", false));
else if(globals->hasProp("noRecover"))
req->setNorecover(globals->getPropBool("noRecover", false));
if(globals->hasProp("connect"))
req->setMaxConnections(globals->getPropInt("connect"));
if(globals->hasProp("throttle"))
req->setThrottle(globals->getPropInt("throttle"));
if(globals->hasProp("transferbuffersize"))
req->setTransferBufferSize(globals->getPropInt("transferbuffersize"));
if(srcxml == NULL)
info("\nFixed spraying from %s on %s to %s\n", srcfile, srcip, dstname);
else
info("\nSpraying to %s\n", dstname);
Owned<IClientSprayFixedResponse> result = sprayclient->SprayFixed(req);
const char *wuid = result->getWuid();
if (!wuid||!*wuid) {
result->getExceptions().errorMessage(except);
return false;
}
retwuid.append(wuid);
return true;
}
void serializeSlaveData(MemoryBuffer &dst, unsigned slave)
{
if (slave < width) // if false - due to mismatch width fitting - fill in with a blank entry
{
dst.append(true);
Owned<IPartDescriptor> originalPartDesc = originalDesc->getPart(slave);
originalPartDesc->serialize(dst);
Owned<IPartDescriptor> newIndexPartDesc = newIndexDesc->getPart(slave);
newIndexPartDesc->serialize(dst);
patchDesc->queryPart(slave)->serialize(dst);
if (0 == slave)
{
if (!local)
{
dst.append(true);
Owned<IPartDescriptor> originalTlkPartDesc = originalDesc->getPart(originalDesc->numParts()-1);
originalTlkPartDesc->serialize(dst);
Owned<IPartDescriptor> newIndexTlkPartDesc = newIndexDesc->getPart(newIndexDesc->numParts()-1);
newIndexTlkPartDesc->serialize(dst);
patchDesc->queryPart(patchDesc->numParts()-1)->serialize(dst);
}
else
dst.append(false);
}
}
else
dst.append(false); // no part
}
bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const char* srcfile,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format,StringBuffer &retwuid, StringBuffer &except)
{
Owned<IClientSprayVariable> req = sprayclient->createSprayVariableRequest();
if(srcxml == NULL)
{
req->setSourceIP(srcip);
req->setSourcePath(srcfile);
}
else
{
req->setSrcxml(xmlbuf);
}
const char* mrsstr = globals->queryProp("maxRecordSize");
if(mrsstr != NULL)
req->setSourceMaxRecordSize(atoi(mrsstr));
else
req->setSourceMaxRecordSize(8192);
const char* encoding = globals->queryProp("encoding");
const char* rowtag = globals->queryProp("rowtag");
if(stricmp(format, "xml") == 0)
{
if(encoding == NULL)
encoding = "utf8";
else if(stricmp(encoding, "ascii") == 0)
throw MakeStringException(-1, "xml format only accepts utf encodings");
if(rowtag == NULL || *rowtag == '\0')
throw MakeStringException(-1, "rowtag not specified.");
}
else if(stricmp(format, "csv") == 0)
{
if(encoding == NULL)
encoding = "ascii";
if(rowtag != NULL && *rowtag != '\0')
throw MakeStringException(-1, "You can't use rowtag option with csv/delimited format");
const char* separator = globals->queryProp("separator");
if(separator && *separator)
req->setSourceCsvSeparate(separator);
const char* terminator = globals->queryProp("terminator");
if(terminator && *terminator)
req->setSourceCsvTerminate(terminator);
const char* quote = globals->queryProp("quote");
if(quote)
{
// Pass quote definition string from command line if defined
// even it is empty to override default value
req->setSourceCsvQuote(quote);
}
}
else
encoding = format; // may need extra later
req->setSourceFormat(CDFUfileformat::decode(encoding));
if(rowtag != NULL)
req->setSourceRowTag(rowtag);
if(dstcluster != NULL)
req->setDestGroup(dstcluster);
req->setDestLogicalName(dstname);
req->setOverwrite(globals->getPropBool("overwrite", false));
req->setReplicate(globals->getPropBool("replicate", true));
req->setReplicateOffset(globals->getPropInt("replicateoffset",1));
if(globals->hasProp("prefix"))
req->setPrefix(globals->queryProp("prefix"));
if(globals->hasProp("nosplit"))
req->setNosplit(globals->getPropBool("nosplit", false));
if(globals->hasProp("push")) {
if (globals->getPropBool("push"))
req->setPush(true);
else
req->setPull(true);
}
if(globals->hasProp("compress"))
req->setCompress(globals->getPropBool("compress", false));
if(globals->hasProp("encrypt"))
req->setEncrypt(globals->queryProp("encrypt"));
if(globals->hasProp("decrypt"))
req->setDecrypt(globals->queryProp("decrypt"));
if(globals->hasProp("norecover"))
req->setNorecover(globals->getPropBool("norecover", false));
else if(globals->hasProp("noRecover"))
req->setNorecover(globals->getPropBool("noRecover", false));
if(globals->hasProp("connect"))
req->setMaxConnections(globals->getPropInt("connect"));
if(globals->hasProp("throttle"))
req->setThrottle(globals->getPropInt("throttle"));
if(globals->hasProp("transferbuffersize"))
req->setTransferBufferSize(globals->getPropInt("transferbuffersize"));
if(globals->hasProp("failIfNoSourceFile"))
req->setFailIfNoSourceFile(globals->getPropBool("failIfNoSourceFile",false));
if(srcxml == NULL)
info("\nVariable spraying from %s on %s to %s\n", srcfile, srcip, dstname);
else
info("\nSpraying to %s\n", dstname);
Owned<IClientSprayResponse> result = sprayclient->SprayVariable(req);
const char *wuid = result->getWuid();
if (!wuid||!*wuid) {
result->getExceptions().errorMessage(except);
return false;
}
retwuid.append(wuid);
return true;
}
void RemoteDataSourceServer::doCmdNumRows(MemoryBuffer & in, MemoryBuffer & out)
{
Owned<IFvDataSource> ds = readDataSource(in);
__int64 numRows = ds ? ds->numRows(true) : 0;
out.append(numRows);
}
int CDfuPlusHelper::copy()
{
const char* srcname = globals->queryProp("srcname");
if(srcname == NULL)
throw MakeStringException(-1, "srcname not specified");
const char* dstname = globals->queryProp("dstname");
if(dstname == NULL)
throw MakeStringException(-1, "dstname not specified");
const char* dstcluster = globals->queryProp("dstcluster");
const char* dstclusterroxie = globals->queryProp("dstclusterroxie");
const char* srcdali = globals->queryProp("srcdali");
const char* srcusername = globals->queryProp("srcusername");
const char* srcpassword = globals->queryProp("srcpassword");
const char* diffkeydst = globals->queryProp("diffkeysrc");
const char* diffkeysrc = globals->queryProp("diffkeydst");
bool nowait = globals->getPropBool("nowait", false);
info("\nCopying from %s to %s\n", srcname, dstname);
Owned<IClientCopy> req = sprayclient->createCopyRequest();
req->setSourceLogicalName(srcname);
req->setDestLogicalName(dstname);
if(dstcluster != NULL)
req->setDestGroup(dstcluster);
if(dstclusterroxie && !stricmp(dstclusterroxie, "Yes"))
req->setDestGroupRoxie("Yes");
if(srcdali != NULL)
req->setSourceDali(srcdali);
if(diffkeysrc != NULL)
req->setSourceDiffKeyName(diffkeysrc);
if(diffkeydst != NULL)
req->setDestDiffKeyName(diffkeydst);
if(srcusername && *srcusername)
req->setSrcusername(srcusername);
if(srcpassword && *srcpassword)
req->setSrcpassword(srcpassword);
req->setOverwrite(globals->getPropBool("overwrite", false));
req->setReplicate(globals->getPropBool("replicate", true));
req->setReplicateOffset(globals->getPropInt("replicateoffset",1));
if(globals->hasProp("nosplit"))
req->setNosplit(globals->getPropBool("nosplit", false));
if(globals->hasProp("push")) {
if (globals->getPropBool("push"))
req->setPush(true);
else
req->setPull(true);
}
if(globals->hasProp("compress"))
req->setCompress(globals->getPropBool("compress", false));
if(globals->hasProp("encrypt"))
req->setEncrypt(globals->queryProp("encrypt"));
if(globals->hasProp("decrypt"))
req->setDecrypt(globals->queryProp("decrypt"));
if(globals->hasProp("connect"))
req->setMaxConnections(globals->getPropInt("connect"));
if(globals->hasProp("throttle"))
req->setThrottle(globals->getPropInt("throttle"));
if(globals->hasProp("transferbuffersize"))
req->setTransferBufferSize(globals->getPropInt("transferbuffersize"));
if(globals->hasProp("wrap"))
req->setWrap(globals->getPropBool("wrap", false));
if(globals->hasProp("multicopy"))
req->setMulticopy(globals->getPropBool("multicopy", false));
else if(globals->hasProp("multiCopy"))
req->setMulticopy(globals->getPropBool("multiCopy", false));
if(globals->hasProp("norecover"))
req->setNorecover(globals->getPropBool("norecover", false));
else if(globals->hasProp("noRecover"))
req->setNorecover(globals->getPropBool("noRecover", false));
Owned<IClientCopyResponse> result = sprayclient->Copy(req);
const char* wuid = result->getResult();
if(wuid == NULL || *wuid == '\0')
exc(result->getExceptions(),"copying");
else
{
const char* jobname = globals->queryProp("jobname");
if(jobname && *jobname)
updatejobname(wuid, jobname);
info("Submitted WUID %s\n", wuid);
if(!nowait)
waitToFinish(wuid);
}
return 0;
}
void CWriteMasterBase::init()
{
published = false;
recordsProcessed = 0;
OwnedRoxieString fname(diskHelperBase->getFileName());
dlfn.set(fname);
if (diskHelperBase->getFlags() & TDWextend)
{
assertex(0 == (diskHelperBase->getFlags() & (TDXtemporary|TDXjobtemp)));
Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), fname, false, true);
if (file.get())
{
fileDesc.setown(file->getFileDescriptor());
queryThorFileManager().noteFileRead(container.queryJob(), file, true);
}
}
if (dlfn.isExternal())
mpTag = container.queryJob().allocateMPTag(); // used
if (NULL == fileDesc.get())
{
bool overwriteok = 0!=(TDWoverwrite & diskHelperBase->getFlags());
unsigned idx=0;
while (true)
{
OwnedRoxieString cluster(diskHelperBase->getCluster(idx));
if(!cluster)
break;
clusters.append(cluster);
idx++;
}
IArrayOf<IGroup> groups;
fillClusterArray(container.queryJob(), fname, clusters, groups);
fileDesc.setown(queryThorFileManager().create(container.queryJob(), fname, clusters, groups, overwriteok, diskHelperBase->getFlags()));
if (1 == groups.ordinality())
targetOffset = getGroupOffset(groups.item(0), container.queryJob().querySlaveGroup());
IPropertyTree &props = fileDesc->queryProperties();
if (diskHelperBase->getFlags() & (TDWowned|TDXjobtemp|TDXtemporary))
props.setPropBool("@owned", true);
if (diskHelperBase->getFlags() & TDWresult)
props.setPropBool("@result", true);
const char *rececl= diskHelperBase->queryRecordECL();
if (rececl&&*rececl)
props.setProp("ECL", rececl);
bool blockCompressed=false;
void *ekey;
size32_t ekeylen;
diskHelperBase->getEncryptKey(ekeylen,ekey);
if (ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
props.setPropBool("@encrypted", true);
blockCompressed = true;
}
else if (0 != (diskHelperBase->getFlags() & TDWnewcompress) || 0 != (diskHelperBase->getFlags() & TDXcompress))
blockCompressed = true;
if (blockCompressed)
props.setPropBool("@blockCompressed", true);
if (TAKdiskwrite == container.getKind() && (0 != (diskHelperBase->getFlags() & TDXtemporary)) && container.queryOwner().queryOwner() && (!container.queryOwner().isGlobal())) // I am in a child query
{ // do early, because this will be local act. and will not come back to master until end of owning graph.
publish();
}
}
}
bool ok() { return exc.get()==NULL; }
static bool physicalPartCopy(IFile *from,const char *tofile, Owned<IException> &exc, StringBuffer *tmpname)
{
StringBuffer tmpnamestr;
if (!tmpname)
tmpname = &tmpnamestr;
tmpname->append(tofile).append("__");
size32_t l = tmpname->length();
genUUID(*tmpname,true); // true for windows
StringAttr uuid(tmpname->str()+l);
tmpname->append(".tmp");
RemoteFilename tmpfn;
tmpfn.setRemotePath(tmpname->str());
//unsigned lastpc;
#ifdef LOG_PART_COPY
PROGLOG("start physicalPartCopy(%s,%s)",from->queryFilename(),tmpname->str());
#endif
try {
recursiveCreateDirectoryForFile(tmpname->str());
while(!asyncCopyFileSection(
uuid,
from,
tmpfn,
(offset_t)-1, // creates file
0,
(offset_t)-1, // all file
NULL,
PHYSICAL_COPY_POLL_TIME)) {
// Abort check TBD
}
}
catch (IException *e) {
EXCLOG(e,"SingleFileCopy: File copy error");
if (exc)
exc.setown(e);
else
e->Release();
}
Owned<IFile> f = createIFile(tmpfn);
if (!exc.get()&&(tmpnamestr.length()!=0)) {
try {
#ifdef LOG_PART_COPY
PROGLOG("physicalPartCopy rename(%s,%s)",tmpname->str(),pathTail(tofile));
#endif
f->rename(pathTail(tofile));
}
catch (IException *e) {
EXCLOG(e,"SingleFileCopy: File rename error");
if (exc)
exc.setown(e);
else
e->Release();
}
}
if (exc.get()) {
try {
f->remove();
}
catch (IException *e) {
// ignore
e->Release();
}
}
#ifdef LOG_PART_COPY
PROGLOG("done physicalPartCopy %s",(exc.get()==NULL)?"OK":"Failed");
#endif
return exc.get()==NULL;
}
int CDfuPlusHelper::remove()
{
StringArray files;
const char* name = globals->queryProp("name");
const char* names = globals->queryProp("names");
const char* namelist = globals->queryProp("namelist");
if(name && *name)
{
files.append(name);
info("\nRemoving %s\n", name);
}
else if(names && *names)
{
info("\nRemoving %s\n", names);
const char* ptr = names;
while(*ptr != '\0')
{
StringBuffer onesub;
while(*ptr != '\0' && *ptr != ',')
{
onesub.append((char)(*ptr));
ptr++;
}
if(onesub.length() > 0)
files.append(onesub.str());
if(*ptr != '\0')
ptr++;
}
}
else if(namelist && *namelist)
{
FILE* f = fopen(namelist, "r");
if (f)
{
char buffer[1024*1];
while(fgets(buffer,sizeof(buffer)-1,f))
{
if(strlen(buffer) == 0)
break;
info("\nRemoving %s\n", buffer);
const char* ptr = buffer;
while(*ptr != '\0')
{
StringBuffer onesub;
while(*ptr != '\0' && *ptr != ',')
{
onesub.append((char)(*ptr));
ptr++;
}
if(onesub.length() > 0)
files.append(onesub.str());
if(*ptr != '\0')
ptr++;
}
}
fclose(f);
}
}
if(files.length() < 1)
throw MakeStringException(-1, "file name not specified");
Owned<IClientDFUArrayActionRequest> req = dfuclient->createDFUArrayActionRequest();
req->setType("Delete");
req->setLogicalFiles(files);
Owned<IClientDFUArrayActionResponse> resp = dfuclient->DFUArrayAction(req);
const IMultiException* excep = &resp->getExceptions();
if(excep != NULL && excep->ordinality() > 0)
{
StringBuffer errmsg;
excep->errorMessage(errmsg);
if(errmsg.length() > 0)
error("%s\n", errmsg.str());
return -1;
}
const char* result = resp->getDFUArrayActionResult();
StringBuffer resultbuf;
if(result != NULL && *result != '\0')
{
if(*result != '<')
resultbuf.append(result);
else
{
bool intag = false;
int i = 0;
char c;
while((c = result[i]) != '\0')
{
if(c == '<')
intag = true;
else if(c == '>')
intag = false;
else if(!intag)
resultbuf.append(c);
i++;
}
}
}
info("%s\n", resultbuf.str());
return 0;
}
int CDfuPlusHelper::superfile(const char* action)
{
const char* superfile = globals->queryProp("superfile");
if(superfile == NULL || *superfile == '\0')
throw MakeStringException(-1, "superfile name is not specified");
if(stricmp(action, "add") == 0 || stricmp(action, "remove") == 0)
{
const char* before = globals->queryProp("before");
StringArray subfiles;
const char* sfprop = globals->queryProp("subfiles");
if(sfprop != NULL && *sfprop != '\0')
{
const char* ptr = sfprop;
while(*ptr != '\0')
{
StringBuffer onesub;
while(*ptr != '\0' && *ptr != ',')
{
onesub.append((char)(*ptr));
ptr++;
}
if(onesub.length() > 0)
subfiles.append(onesub.str());
if(*ptr != '\0')
ptr++;
}
}
Owned<IClientSuperfileActionRequest> req = dfuclient->createSuperfileActionRequest();
req->setAction(action);
req->setSuperfile(superfile);
req->setSubfiles(subfiles);
req->setBefore(before);
req->setDelete(globals->getPropBool("delete", false));
Owned<IClientSuperfileActionResponse> resp = dfuclient->SuperfileAction(req);
const IMultiException* excep = &resp->getExceptions();
if(excep != NULL && excep->ordinality() > 0)
{
StringBuffer errmsg;
excep->errorMessage(errmsg);
info("%s\n", errmsg.str());
return -1;
}
if(stricmp(action, "add") == 0)
{
info("Addsuper successfully finished");
}
else if(stricmp(action, "remove") == 0)
{
info("Removesuper successfully finished");
}
}
else if(stricmp(action, "list") == 0)
{
Owned<IClientSuperfileListRequest> req = dfuclient->createSuperfileListRequest();
req->setSuperfile(superfile);
Owned<IClientSuperfileListResponse> resp = dfuclient->SuperfileList(req);
const IMultiException* excep = &resp->getExceptions();
if(excep != NULL && excep->ordinality() > 0)
{
StringBuffer errmsg;
excep->errorMessage(errmsg);
error("%s\n", errmsg.str());
return -1;
}
StringArray& result = resp->getSubfiles();
info("%s has %d subfiles:\n", superfile, result.length());
for(unsigned i = 0; i < result.length(); i++)
{
info("\t%s\n", result.item(i));
}
}
return 0;
}
void testMultiCluster()
{
Owned<IGroup> grp1 = createIGroup("192.168.51.1-5");
Owned<IGroup> grp2 = createIGroup("192.168.16.1-5");
Owned<IGroup> grp3 = createIGroup("192.168.53.1-5");
queryNamedGroupStore().add("testgrp1",grp1);
queryNamedGroupStore().add("testgrp2",grp2);
queryNamedGroupStore().add("testgrp3",grp3);
Owned<IFileDescriptor> fdesc = createFileDescriptor();
fdesc->setDefaultDir("/c$/thordata/test");
fdesc->setPartMask("testfile1._$P$_of_$N$");
fdesc->setNumParts(5);
ClusterPartDiskMapSpec mapping;
fdesc->addCluster(grp1,mapping);
fdesc->addCluster(grp2,mapping);
fdesc->addCluster(grp3,mapping);
queryDistributedFileDirectory().removeEntry("test::testfile1",UNKNOWN_USER);
Owned<IDistributedFile> file = queryDistributedFileDirectory().createNew(fdesc);
queryDistributedFileDirectory().removeEntry("test::testfile1",UNKNOWN_USER);
file->attach("test::testfile1",UNKNOWN_USER);
StringBuffer name;
unsigned i;
for (i=0;i<file->numClusters();i++)
PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
file.clear();
file.setown(queryDistributedFileDirectory().lookup("test::testfile1",UNKNOWN_USER));
for (i=0;i<file->numClusters();i++)
PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
file.clear();
file.setown(queryDistributedFileDirectory().lookup("test::testfile1@testgrp1",UNKNOWN_USER));
for (i=0;i<file->numClusters();i++)
PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
file.clear();
queryDistributedFileDirectory().removePhysical("test::testfile1@testgrp2",UNKNOWN_USER);
file.setown(queryDistributedFileDirectory().lookup("test::testfile1",UNKNOWN_USER));
for (i=0;i<file->numClusters();i++)
PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
}
int CDfuPlusHelper::add()
{
const char* lfn = globals->queryProp("dstname");
if(lfn == NULL || *lfn == '\0')
throw MakeStringException(-1, "dstname not specified");
bool isRemote = false;
const char* xmlfname = globals->queryProp("srcxml");
const char* srcname = globals->queryProp("srcname");
const char* srcdali = globals->queryProp("srcdali");
const char* srcusername = globals->queryProp("srcusername");
const char* srcpassword = globals->queryProp("srcpassword");
if(xmlfname == NULL || *xmlfname == '\0')
{
if(srcname == NULL || *srcname == '\0')
throw MakeStringException(-1, "Please specify srcxml for adding from xml, or srcname for adding from remote dali");
else
{
isRemote = true;
if(srcdali == NULL || *srcdali == '\0')
throw MakeStringException(-1, "srcdali not specified for adding remote");
}
}
if(!isRemote)
{
MemoryBuffer xmlbuf;
StringBuffer buf;
buf.loadFile(xmlfname);
int len = buf.length();
xmlbuf.setBuffer(len, buf.detach(), true);
Owned<IClientAddRequest> req = dfuclient->createAddRequest();
req->setDstname(lfn);
req->setXmlmap(xmlbuf);
Owned<IClientAddResponse> resp = dfuclient->Add(req);
const IMultiException* excep = &resp->getExceptions();
if(excep != NULL && excep->ordinality() > 0)
{
StringBuffer errmsg;
excep->errorMessage(errmsg);
error("%s\n", errmsg.str());
return -1;
}
}
else
{
Owned<IClientAddRemoteRequest> req = dfuclient->createAddRemoteRequest();
req->setDstname(lfn);
req->setSrcname(srcname);
req->setSrcdali(srcdali);
if(srcusername != NULL)
req->setSrcusername(srcusername);
if(srcpassword != NULL)
req->setSrcpassword(srcpassword);
Owned<IClientAddRemoteResponse> resp = dfuclient->AddRemote(req);
const IMultiException* excep = &resp->getExceptions();
if(excep != NULL && excep->ordinality() > 0)
{
StringBuffer errmsg;
excep->errorMessage(errmsg);
error("%s\n", errmsg.str());
return -1;
}
}
info("%s successfully added", lfn);
return 0;
}
void processMessage(CMessageBuffer &mb)
{
ICoven &coven=queryCoven();
MemoryBuffer params;
params.swapWith(mb);
int fn;
params.read(fn);
switch (fn) {
case MDR_GET_VALUE: {
StringAttr id;
StringBuffer buf;
params.read(id);
if (0 == stricmp(id,"threads")) {
mb.append(getThreadList(buf).str());
}
else if (0 == stricmp(id, "mpqueue")) {
mb.append(getReceiveQueueDetails(buf).str());
}
else if (0 == stricmp(id, "locks")) { // Legacy - newer diag clients should use querySDS().getLocks() directly
Owned<ILockInfoCollection> lockInfoCollection = querySDS().getLocks();
mb.append(lockInfoCollection->toString(buf).str());
}
else if (0 == stricmp(id, "sdsstats")) { // Legacy - newer diag clients should use querySDS().getUsageStats() directly
mb.append(querySDS().getUsageStats(buf).str());
}
else if (0 == stricmp(id, "connections")) { // Legacy - newer diag clients should use querySDS().getConnections() directly
mb.append(querySDS().getConnections(buf).str());
}
else if (0 == stricmp(id, "sdssubscribers")) { // Legacy - newer diag clients should use querySDS().getSubscribers() directly
mb.append(querySDS().getSubscribers(buf).str());
}
else if (0 == stricmp(id, "clients")) {
mb.append(querySessionManager().getClientProcessList(buf).str());
}
else if (0 == stricmp(id, "subscriptions")) {
mb.append(getSubscriptionList(buf).str());
}
else if (0 == stricmp(id, "mpverify")) {
queryWorldCommunicator().verifyAll(buf);
mb.append(buf.str());
}
else if (0 == stricmp(id, "extconsistency")) {
mb.append(querySDS().getExternalReport(buf).str());
}
else if (0 == stricmp(id, "build")) {
mb.append("$Id: dadiags.cpp 62376 2011-02-04 21:59:58Z sort $");
}
else if (0 == stricmp(id, "sdsfetch")) {
StringAttr branchpath;
params.read(branchpath);
Linked<IPropertyTree> sroot = querySDSServer().lockStoreRead();
try { sroot->queryPropTree(branchpath)->serialize(mb); }
catch (...) { querySDSServer().unlockStoreRead(); throw; }
querySDSServer().unlockStoreRead();
}
else if (0 == stricmp(id, "perf")) {
getSystemTraceInfo(buf,PerfMonStandard);
mb.append(buf.str());
}
else if (0 == stricmp(id, "sdssize")) {
StringAttr branchpath;
params.read(branchpath);
Linked<IPropertyTree> sroot = querySDSServer().lockStoreRead();
StringBuffer sbuf;
try {
toXML(sroot->queryPropTree(branchpath),sbuf);
DBGLOG("sdssize '%s' = %d",branchpath.get(),sbuf.length());
}
catch (...) {
querySDSServer().unlockStoreRead();
throw;
}
querySDSServer().unlockStoreRead();
mb.append(sbuf.length());
}
else if (0 == stricmp(id, "disconnect")) {
StringAttr client;
params.read(client);
SocketEndpoint ep(client);
PROGLOG("Dalidiag request to close client connection: %s", client.get());
Owned<INode> node = createINode(ep);
queryCoven().disconnect(node);
}
else if (0 == stricmp(id, "unlock")) {
__int64 connectionId;
bool disconnect;
params.read(connectionId);
params.read(disconnect);
PROGLOG("Dalidiag request to unlock connection id: %" I64F "x", connectionId);
StringBuffer connectionInfo;
bool success = querySDSServer().unlock(connectionId, disconnect, connectionInfo);
mb.append(success);
if (success)
mb.append(connectionInfo);
}
else if (0 == stricmp(id, "save")) {
PROGLOG("Dalidiag requests SDS save");
querySDSServer().saveRequest();
}
else if (0 == stricmp(id, "settracetransactions")) {
PROGLOG("Dalidiag requests Trace Transactions");
if(traceAllTransactions(true))
mb.append("OK - no change");
else
mb.append("OK - transaction tracing enabled");
}
else if (0 == stricmp(id, "cleartracetransactions")) {
PROGLOG("Dalidiag requests Trace Transactions stopped");
if(traceAllTransactions(false))
mb.append("OK - transaction tracing disabled");
else
mb.append("OK - no change");
}
else if (0 == stricmp(id, "setldapflags")) {
unsigned f;
params.read(f);
PROGLOG("Dalidiag requests setldapflags %d",f);
querySessionManager().setLDAPflags(f);
}
else if (0 == stricmp(id, "getldapflags")) {
unsigned f=querySessionManager().getLDAPflags();;
mb.append(f);
}
else if (0 == stricmp(id, "setsdsdebug")) {
PROGLOG("Dalidiag setsdsdebug");
unsigned p;
params.read(p);
StringArray arr;
while (p--)
{
StringAttr s;
params.read(s);
arr.append(s);
}
StringBuffer reply;
bool success = querySDSServer().setSDSDebug(arr, reply);
mb.append(success).append(reply);
}
else
mb.append(StringBuffer("UNKNOWN OPTION: ").append(id).str());
}
break;
}
coven.reply(mb);
}
int CDfuPlusHelper::waitToFinish(const char* wuid)
{
if(wuid == NULL || *wuid == '\0')
return 0;
Owned<IClientGetDFUWorkunit> req = sprayclient->createGetDFUWorkunitRequest();
req->setWuid(wuid);
int wait_cycle = 10;
while(true)
{
Owned<IClientGetDFUWorkunitResponse> resp = sprayclient->GetDFUWorkunit(req);
const IMultiException* excep = &resp->getExceptions();
if(excep != NULL && excep->ordinality() > 0)
{
StringBuffer errmsg;
excep->errorMessage(errmsg);
error("%s\n", errmsg.str());
break;
}
IConstDFUWorkunit & dfuwu = resp->getResult();
switch(dfuwu.getState())
{
case DFUstate_unknown:
info("%s status: unknown\n", wuid);
break;
case DFUstate_scheduled:
info("%s status: scheduled\n", wuid);
break;
case DFUstate_queued:
info("%s status: queued\n", wuid);
break;
case DFUstate_started:
info("%s\n", dfuwu.getProgressMessage());
break;
case DFUstate_aborting:
info("%s aborting\n", wuid);
break;
case DFUstate_aborted:
info("%s aborted\n", wuid);
return -1;
case DFUstate_failed:
{
const char* errmsg = dfuwu.getSummaryMessage();
if(errmsg && *errmsg)
info("%s\n", errmsg);
else
info("%s failed.\n", wuid);
return -1;
}
case DFUstate_finished:
info("%s Finished\n", wuid);
info("%s\n", dfuwu.getSummaryMessage());
return 0;
}
sleep(wait_cycle);
// make it wait shorter time at the beginning to avoid waiting too long for short jobs.
if(wait_cycle < WAIT_SECONDS)
wait_cycle = (wait_cycle+10<WAIT_SECONDS)?(wait_cycle+10):WAIT_SECONDS;
}
return 0;
}
IPropertyTree* CFileSpraySoapBindingEx::createPTreeForXslt(double clientVersion, const char* method, const char* dfuwuid)
{
Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
Owned<IConstEnvironment> constEnv = factory->openEnvironment();
Owned<IPropertyTree> pEnvRoot = &constEnv->getPTree();
IPropertyTree* pEnvSoftware = pEnvRoot->queryPropTree("Software");
Owned<IPropertyTree> pRoot = createPTreeFromXMLString("<Environment/>");
IPropertyTree* pSoftware = pRoot->addPropTree("Software", createPTree("Software"));
if (pEnvSoftware)
{
StringBuffer dfuwuidSourcePartIP, wuxml;
if(dfuwuid && *dfuwuid)
{
Owned<IDFUWorkUnitFactory> dfuwu_factory = getDFUWorkUnitFactory();
Owned<IConstDFUWorkUnit> dfuwu = dfuwu_factory->openWorkUnit(dfuwuid, false);
if(dfuwu)
{
dfuwu->toXML(wuxml);
Owned<IPropertyTree> wu = createPTreeFromXMLString(wuxml.str());
if (wu)
{
const char* ip = wu->queryProp("Source/Part/@node");
if (ip && *ip)
dfuwuidSourcePartIP.append(ip);
}
}
}
appendDropZones(clientVersion, constEnv, dfuwuidSourcePartIP.str(), pSoftware);
//For Spray files on Thor Cluster, fetch all the group names for all the thor instances (and dedup them)
BoolHash uniqueThorClusterGroupNames;
Owned<IPropertyTreeIterator> it =pEnvSoftware->getElements("ThorCluster");
ForEach(*it)
{
StringBuffer thorClusterGroupName;
IPropertyTree& cluster = it->query();
getClusterGroupName(cluster, thorClusterGroupName);
if (!thorClusterGroupName.length())
continue;
bool* found = uniqueThorClusterGroupNames.getValue(thorClusterGroupName.str());
if (found && *found)
continue;
uniqueThorClusterGroupNames.setValue(thorClusterGroupName.str(), true);
IPropertyTree* newClusterTree = pSoftware->addPropTree("ThorCluster", &it->get());
newClusterTree->setProp("@name", thorClusterGroupName.str()); //set group name into @name for spray target
}
it.setown(pEnvSoftware->getElements("EclAgentProcess"));
ForEach(*it)
{
IPropertyTree &cluster = it->query();
const char* name = cluster.queryProp("@name");
if (!name||!*name)
continue;
unsigned ins = 0;
Owned<IPropertyTreeIterator> insts = cluster.getElements("Instance");
ForEach(*insts)
{
const char *na = insts->query().queryProp("@netAddress");
if (!na || !*na)
{
insts->query().setProp("@gname", name);
continue;
}
SocketEndpoint ep(na);
if (ep.isNull())
continue;
ins++;
StringBuffer gname("hthor__");
//StringBuffer gname;
gname.append(name);
if (ins>1)
gname.append('_').append(ins);
insts->query().setProp("@gname", gname.str());
}
pSoftware->addPropTree("EclAgentProcess", &it->get());
}
if (stricmp(method, "CopyInput") == 0) //Limit for this method only
{
it.setown(pEnvSoftware->getElements("RoxieCluster"));
ForEach(*it)
pSoftware->addPropTree("RoxieCluster", &it->get());
}
if (wuxml.length() > 0)
pSoftware->addPropTree("DfuWorkunit", createPTreeFromXMLString(wuxml.str()));
}
return pRoot.getClear();
}
IEspRpcBinding *queryBinding()
{
return binding_.get();
}