void CEclAgentExecutionServer::start(StringBuffer & codeDir)
{
if (started)
{
WARNLOG("START called when already started\n");
assert(false);
}
codeDirectory = codeDir;
StringBuffer propertyFile = codeDirectory;
addPathSepChar(propertyFile);
propertyFile.append("agentexec.xml");
Owned<IPropertyTree> properties;
try
{
DBGLOG("AgentExec: Loading properties file '%s'\n", propertyFile.str());
properties.setown(createPTreeFromXMLFile(propertyFile.str()));
}
catch (IException *e)
{
EXCLOG(e, "Error processing properties file\n");
throwUnexpected();
}
{
//Build logfile from component properties settings
Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(properties, "eclagent");
lf->setCreateAliasFile(false);
lf->setMsgFields(MSGFIELD_timeDate | MSGFIELD_msgID | MSGFIELD_process | MSGFIELD_thread | MSGFIELD_code);
lf->beginLogging();
PROGLOG("Logging to %s",lf->queryLogFileSpec());
}
//get name of workunit job queue
StringBuffer sb;
properties->getProp("@name", sb.clear());
agentName.set(sb);
if (!agentName.length())
{
ERRLOG("'name' not specified in properties file\n");
throwUnexpected();
}
//get dali server(s)
properties->getProp("@daliServers", daliServers);
if (!daliServers.length())
{
ERRLOG("'daliServers' not specified in properties file\n");
throwUnexpected();
}
started = true;
Thread::start();
Thread::join();
}
void CEclAgentExecutionServer::start()
{
if (started)
{
WARNLOG("START called when already started\n");
assert(false);
}
Owned<IPropertyTree> properties;
try
{
DBGLOG("AgentExec: Loading properties file 'agentexec.xml'");
properties.setown(createPTreeFromXMLFile("agentexec.xml"));
}
catch (IException *e)
{
EXCLOG(e, "Error processing properties file\n");
throwUnexpected();
}
{
//Build logfile from component properties settings
Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(properties, "eclagent");
lf->setCreateAliasFile(false);
lf->beginLogging();
PROGLOG("Logging to %s",lf->queryLogFileSpec());
}
//get name of workunit job queue
StringBuffer sb;
properties->getProp("@name", sb.clear());
agentName.set(sb);
if (!agentName.length())
{
ERRLOG("'name' not specified in properties file\n");
throwUnexpected();
}
setStatisticsComponentName(SCThthor, agentName, true);
//get dali server(s)
properties->getProp("@daliServers", daliServers);
if (!daliServers.length())
{
ERRLOG("'daliServers' not specified in properties file\n");
throwUnexpected();
}
started = true;
Thread::start();
Thread::join();
}
const void *nextRowGENoCatch(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
{
if (!eos)
{
try
{
ActivityTimer t(totalCycles, timeActivities, NULL);
OwnedConstThorRow ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
if (ret && wasCompleteMatch)
dataLinkIncrement();
return ret.getClear();
}
catch (IException *e)
{
eos = true;
ActPrintLog(e, NULL);
e->Release();
helper->onExceptionCaught();
}
catch (...)
{
eos = true;
helper->onExceptionCaught();
}
throwUnexpected(); // onExceptionCaught should have thrown something
}
return NULL;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities, NULL);
if (!eos)
{
try
{
OwnedConstThorRow row(input->nextRow());
if (!row)
return NULL;
dataLinkIncrement();
return row.getClear();
}
catch (IException *e)
{
eos = true;
ActPrintLog(e, NULL);
e->Release();
helper->onExceptionCaught();
}
catch (...)
{
eos = true;
helper->onExceptionCaught();
}
throwUnexpected(); // onExceptionCaught should have thrown something
}
return NULL;
}
unsigned DataSourceMetaData::numKeyedColumns() const
{
unsigned count = 0;
unsigned curOffset = 0;
ForEachItemIn(i, fields)
{
if (curOffset >= keyedSize)
break;
DataSourceMetaItem & cur = fields.item(i);
switch (cur.flags)
{
case FVFFnone:
{
unsigned size = cur.type->getSize();
assertex(size != UNKNOWN_LENGTH);
curOffset += size;
count++;
break;
}
default:
throwUnexpected();
}
}
return count;
assertex(curOffset == keyedSize);
}
void DataSourceMetaData::extractKeyedInfo(UnsignedArray & offsets, TypeInfoArray & types)
{
unsigned curOffset = 0;
ForEachItemIn(i, fields)
{
if (curOffset >= keyedSize)
break;
DataSourceMetaItem & cur = fields.item(i);
switch (cur.flags)
{
case FVFFnone:
{
offsets.append(curOffset);
types.append(*LINK(cur.type));
unsigned size = cur.type->getSize();
assertex(size != UNKNOWN_LENGTH);
curOffset += size;
break;
}
case FVFFbeginrecord:
case FVFFendrecord:
break;
default:
throwUnexpected();
}
}
offsets.append(curOffset);
assertex(curOffset == keyedSize);
}
virtual void main()
{
running = true;
loop
{
INode *senderNode;
CMessageBuffer msg;
if (!queryWorldCommunicator().recv(msg, NULL, MPTAG_THORREGISTRATION, &senderNode))
return;
rank_t sender = queryClusterGroup().rank(senderNode);
SocketEndpoint ep = senderNode->endpoint();
ep.port -= THOR_MP_INC;
StringBuffer url;
ep.getUrlStr(url);
if (RANK_NULL == sender)
{
PROGLOG("Node %s trying to deregister is not part of this cluster", url.str());
continue;
}
RegistryCode code;
msg.read((int &)code);
if (!rc_deregister == code)
throwUnexpected();
registry.deregisterNode(sender);
}
running = false;
}
extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result)
{
bool hasVersion = len && (*(const byte *)data == 0x80);
MemoryBuffer src;
src.setBuffer(len, const_cast<void *>(data), false);
byte version = 1;
if (hasVersion)
{
src.skip(1);
src.read(version);
}
MemoryBuffer tgt;
switch (version)
{
case 1:
decompressToBuffer(tgt, src);
break;
default:
throwUnexpected();
}
tgt.append((char)0);
unsigned expandedLen = tgt.length();
result.setBuffer(expandedLen, reinterpret_cast<char *>(tgt.detach()), expandedLen-1);
return true;
}
virtual void main()
{
running = true;
loop
{
INode *senderNode;
CMessageBuffer msg;
if (!queryWorldCommunicator().recv(msg, NULL, MPTAG_THORREGISTRATION, &senderNode))
return;
rank_t sender = queryNodeGroup().rank(senderNode);
SocketEndpoint ep = senderNode->endpoint();
StringBuffer url;
ep.getUrlStr(url);
if (RANK_NULL == sender)
{
PROGLOG("Node %s trying to deregister is not part of this cluster", url.str());
continue;
}
RegistryCode code;
msg.read((int &)code);
if (rc_deregister != code)
throwUnexpected();
Owned<IException> e = deserializeException(msg);
if (e.get())
EXCLOG(e, "Slave unregistered with exception");
registry.deregisterNode(sender-1);
}
running = false;
}
static bool getResourceFromMappedFile(const char * filename, const byte * start_addr, MemoryBuffer &data, const char * type, unsigned id)
{
#if defined(_WIN32) || defined (_USE_BINUTILS)
throwUnexpected();
#elif defined(__APPLE__)
VStringBuffer sectname("%s_%u", type, id);
// The first bytes are the Mach-O header
const struct mach_header_64 *mh = (const struct mach_header_64 *) start_addr;
if (mh->magic != MH_MAGIC_64)
{
DBGLOG("Failed to extract resource %s: Does not appear to be a Mach-O 64-bit binary", filename);
return false;
}
unsigned long len = 0;
unsigned char *data2 = getsectiondata(mh, "__TEXT", sectname.str(), &len);
data.append(len, data2);
return true;
#else
// The first bytes are the ELF header
const Elf64_Ehdr * hdr = (const Elf64_Ehdr *) start_addr;
if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0)
{
DBGLOG("Failed to extract resource %s: Does not appear to be a ELF binary", filename);
return false;
}
if (hdr->e_ident[EI_CLASS] != ELFCLASS64)
{
DBGLOG("Failed to extract resource %s: Does not appear to be a ELF 64-bit binary", filename);
return false;
}
//Check that there is a symbol table for the sections.
if (hdr->e_shstrndx == SHN_UNDEF)
{
DBGLOG("Failed to extract resource %s: Does not include a section symbol table", filename);
return false;
}
//Now walk the sections comparing the section names
Elf64_Half numSections = hdr->e_shnum;
const Elf64_Shdr * sectionHeaders = reinterpret_cast<const Elf64_Shdr *>(start_addr + hdr->e_shoff);
const Elf64_Shdr & symbolTableSection = sectionHeaders[hdr->e_shstrndx];
const char * symbolTable = (const char *)start_addr + symbolTableSection.sh_offset;
VStringBuffer sectname("%s_%u", type, id);
for (unsigned iSect= 0; iSect < numSections; iSect++)
{
const Elf64_Shdr & section = sectionHeaders[iSect];
const char * sectionName = symbolTable + section.sh_name;
if (streq(sectionName, sectname))
{
data.append(section.sh_size, start_addr + section.sh_offset);
return true;
}
}
DBGLOG("Failed to extract resource %s: Does not include a matching entry", filename);
return false;
#endif
}
void CThorStreamDeserializerSource::skipPackedInt()
{
throwUnexpected();
size32_t available;
const byte * temp = doPeek(1, available);
size32_t size = rtlGetPackedSizeFromFirst(*temp);
in->skip(size);
}
static bool isShared(const void *ptr)
{
if (ptr)
{
HeapletBase *h = findBase(ptr);
return h->_isShared(ptr);
}
// isShared(NULL) or isShared on an object that shares a link-count is an error
throwUnexpected();
}
static size32_t capacity(const void *ptr)
{
if (ptr)
{
HeapletBase *h = findBase(ptr);
//MORE: If capacity was always the size stored in the first word of the block this could be non virtual
//and the whole function could be inline.
return h->_capacity();
}
throwUnexpected();
}
static void addElementToPTree(IPropertyTree * root, IDefRecordElement * elem)
{
byte kind = elem ? elem->getKind() : DEKnone;
Owned<IPTree> branch = createPTree();
StringAttr branchName;
switch (kind)
{
case DEKnone:
branchName.set("None");
assertex(elem->numChildren() == 0);
break;
case DEKrecord:
{
branchName.set("Record");
branch->setPropInt("@maxSize", elem->getMaxSize());
unsigned numChildren = elem->numChildren();
for (unsigned i=0; i < numChildren; i++)
addElementToPTree(branch, elem->queryChild(i));
break;
}
case DEKifblock:
{
branchName.set("IfBlock");
StringBuffer value;
elem->queryCompareValue()->getStringValue(value);
branch->setProp("@compareValue", value.str());
assertex(elem->numChildren() == 2);
addElementToPTree(branch, elem->queryChild(0));
addElementToPTree(branch, elem->queryChild(0));
break;
}
case DEKfield:
{
branchName.set("Field");
branch->setProp("@name", elem->queryName()->str());
branch->setPropInt("@maxSize", elem->getMaxSize());
StringBuffer type;
elem->queryType()->getDescriptiveType(type);
branch->setProp("@type", type.str());
assertex(elem->numChildren() <= 1);
if(elem->numChildren())
addElementToPTree(branch, elem->queryChild(0));
break;
}
default:
throwUnexpected();
}
root->addPropTree(branchName.get(), branch.getClear());
}
void init()
{
StringBuffer xpath("Software/ThorCluster[@name=\"");
xpath.append(clusterName).append("\"]");
Owned<IRemoteConnection> conn = querySDS().connect("/Environment", myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
environment.setown(createPTreeFromIPT(conn->queryRoot()));
options = environment->queryPropTree(xpath.str());
if (!options)
throwUnexpected();
groupName.set(options->queryProp("@nodeGroup"));
if (groupName.isEmpty())
groupName.set(options->queryProp("@name"));
VStringBuffer spareS("%s_spares", groupName.get());
spareGroupName.set(spareS);
group.setown(queryNamedGroupStore().lookup(groupName));
spareGroup.setown(queryNamedGroupStore().lookup(spareGroupName));
}
void RowTransformer::createRowRecord(FieldMapping const & mapping, CIArrayOf<RowRecord> & records, size32_t diskOffset, unsigned numVarFields, bool & prevActivityField, unsigned & prevActivityFieldNum)
{
size32_t diskSize = mapping.queryDiskFieldSize();
unsigned activityFieldNum = mapping.queryActivityFieldNum();
switch(mapping.queryType())
{
case FieldMapping::Simple:
if(mapping.isDiskFieldFpos())
if(mapping.isActivityFieldFpos())
{
ensureItem(records, activityFieldNum).setVals(0, 0, diskSize, false).setFpos(true, true);
prevActivityField = false;
}
else
{
ensureItem(records, activityFieldNum).setVals(0, 0, diskSize, false).setFpos(false, true);
prevActivityField = false;
}
else
if(mapping.isActivityFieldFpos())
{
ensureItem(records, activityFieldNum).setVals(diskOffset, numVarFields, diskSize, false).setFpos(true, false);
prevActivityField = false;
}
else
{
ensureItem(records, activityFieldNum).setVals(diskOffset, numVarFields, diskSize, (prevActivityField && (activityFieldNum == (prevActivityFieldNum+1))));
prevActivityField = true;
prevActivityFieldNum = activityFieldNum;
}
break;
case FieldMapping::ChildDataset:
ensureItem(records, activityFieldNum).setVals(diskOffset, numVarFields, diskSize, false).setChildMappings(&mapping.queryChildMappings());
prevActivityField = false;
break;
case FieldMapping::None:
prevActivityField = false;
break;
default:
throwUnexpected();
}
}
static unsigned __int64 getDateTimeValue(const MYSQL_BIND &bound)
{
const MYSQL_TIME * time = (const MYSQL_TIME *) bound.buffer;
switch (bound.buffer_type)
{
case MYSQL_TYPE_TIMESTAMP:
case MYSQL_TYPE_DATETIME:
//What format should this be? Possibly a timestamp_t
return (unsigned __int64)((time->year * 10000) + (time->month * 100) + (time->day)) * 1000000 +
(time->hour * 10000) + (time->minute * 100) + (time->second);
case MYSQL_TYPE_DATE:
return (time->year * 10000) + (time->month * 100) + (time->day);
case MYSQL_TYPE_TIME:
return (time->hour * 10000) + (time->minute * 100) + (time->second);
default:
throwUnexpected();
}
}
IHqlExpression * HqlCppCaseInfo::buildIndexedMap(BuildCtx & ctx, IHqlExpression * test, unsigned lower, unsigned upper)
{
ITypeInfo * compareType = test->queryType()->queryPromotedType();
type_t compareTypeCode = compareType->getTypeCode();
HqlExprArray values;
IHqlExpression * dft = queryActiveTableSelector(); // value doesn't matter as long as it will not occur
unsigned num = (upper-lower+1);
values.ensure(num);
unsigned idx;
for (idx = 0; idx < num; idx++)
values.append(*LINK(dft));
ForEachItemIn(idx2, pairs)
{
IHqlExpression & cur = pairs.item(idx2);
IValue * value = cur.queryChild(0)->queryValue();
unsigned replaceIndex;
switch (compareTypeCode)
{
case type_int:
replaceIndex = (int)value->getIntValue()-lower;
break;
case type_string:
{
StringBuffer temp;
value->getStringValue(temp);
replaceIndex = (int)(unsigned char)temp.charAt(0)-lower;
break;
}
default:
throwUnexpectedType(compareType);
}
IHqlExpression * mapTo = cur.queryChild(1);
if (mapTo->getOperator() != no_constant)
throwUnexpected();
if (replaceIndex >= num)
translator.reportWarning(CategoryIgnored, HQLWRN_CaseCanNeverMatch, "CASE entry %d can never match the test condition", replaceIndex);
else
values.replace(*LINK(mapTo),replaceIndex);
}
void start()
{
ActivityTimer s(totalCycles, timeActivities, NULL);
dataLinkStart();
eogPending = false;
if (container.queryLocal() || firstNode())
{
CMessageBuffer reqMsg;
reqMsg.setReplyTag(replyTag);
reqMsg.append(smt_actMsg);
reqMsg.append(container.queryOwner().queryGraphId());
reqMsg.append(container.queryId());
if (!container.queryJob().queryJobComm().sendRecv(reqMsg, 0, container.queryJob().querySlaveMpTag(), LONGTIMEOUT))
throwUnexpected();
masterReplyMsg.swapWith(reqMsg);
}
}
void CThorStreamDeserializerSource::skipUtf8(size32_t len)
{
throwUnexpected();
loop
{
if (len == 0)
return;
size32_t available;
const byte * cur = doPeek(1, available);
size32_t copyLen;
for (copyLen = 0; copyLen < available;)
{
copyLen += readUtf8Size(cur+copyLen); // This function only accesses the first byte
len--;
}
in->skip(copyLen);
}
}
static void getDateTimeText(const MYSQL_BIND &bound, size32_t &chars, char * &result)
{
const MYSQL_TIME * time = (const MYSQL_TIME *) bound.buffer;
char temp[20];
switch (bound.buffer_type)
{
case MYSQL_TYPE_TIMESTAMP:
case MYSQL_TYPE_DATETIME:
_snprintf(temp, sizeof(temp), "%4u-%02u-%02u %02u:%02u:%02u", time->year, time->month, time->day, time->hour, time->minute, time->second);
break;
case MYSQL_TYPE_DATE:
_snprintf(temp, sizeof(temp), "%4u-%02u-%02u", time->year, time->month, time->day);
break;
case MYSQL_TYPE_TIME:
_snprintf(temp, sizeof(temp), "%02u:%02u:%02u", time->hour, time->minute, time->second);
break;
default:
throwUnexpected();
}
rtlStrToStrX(chars, result, strlen(temp), temp);
}
static void serializeElement(MemoryBuffer & target, IDefRecordElement * elem)
{
byte kind = elem ? elem->getKind() : DEKnone;
target.append(kind);
switch (kind)
{
case DEKnone:
break;
case DEKrecord:
{
size32_t maxSize = elem->getMaxSize();
unsigned numChildren = elem->numChildren();
target.append(maxSize).append(numChildren);
for (unsigned i=0; i < numChildren; i++)
serializeElement(target, elem->queryChild(i));
break;
}
case DEKifblock:
{
IValue * value = elem->queryCompareValue();
serializeValue(target, value);
serializeElement(target, elem->queryChild(0));
serializeElement(target, elem->queryChild(1));
break;
}
case DEKfield:
{
_ATOM name = elem->queryName();
ITypeInfo * type = elem->queryType();
size32_t maxSize = elem->getMaxSize();
serializeAtom(target, name);
type->serialize(target);
serializeElement(target, elem->queryChild(0));
target.append(maxSize);
break;
}
default:
throwUnexpected();
}
}
static IDefRecordElement * deserializeElement(MemoryBuffer & source)
{
byte kind;
source.read(kind);
switch (kind)
{
case DEKnone:
return NULL;
case DEKrecord:
{
size32_t maxSize;
unsigned numChildren;
source.read(maxSize).read(numChildren);
Owned<IDefRecordBuilder> builder = createDErecord(maxSize);
for (unsigned i=0; i < numChildren; i++)
builder->addChildOwn(deserializeElement(source));
return builder->close();
}
case DEKifblock:
{
Owned<IValue> value = deserializeValue(source);
Owned<IDefRecordElement> field = deserializeElement(source);
Owned<IDefRecordElement> record = deserializeElement(source);
return createDEifblock(field, value, record);
}
case DEKfield:
{
_ATOM name = deserializeAtom(source);
Owned<ITypeInfo> type = deserializeType(source);
Owned<IDefRecordElement> record = deserializeElement(source);
size32_t maxSize;
source.read(maxSize);
return createDEfield(name, type, record, maxSize);
}
default:
throwUnexpected();
}
}
extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, MemoryBuffer &result)
{
bool hasVersion = len && (*(const byte *)data == 0x80);
MemoryBuffer src;
src.setBuffer(len, const_cast<void *>(data), false);
byte version = 1;
if (hasVersion)
{
src.skip(1);
src.read(version);
}
switch (version)
{
case 1:
decompressToBuffer(result, src);
break;
default:
throwUnexpected();
}
return true;
}
//IThorIndexCallback
virtual unsigned __int64 getFilePosition(const void *row)
{
throwUnexpected();
}
byte * RtlStaticRowBuilder::createSelf()
{
throwUnexpected();
}
void CEclAgentExecutionServer::start(StringBuffer & codeDir)
{
if (started)
{
WARNLOG("START called when already started\n");
assert(false);
}
codeDirectory = codeDir;
StringBuffer propertyFile = codeDirectory;
addPathSepChar(propertyFile);
propertyFile.append("agentexec.xml");
Owned<IPropertyTree> properties;
try
{
DBGLOG("AgentExec: Loading properties file '%s'\n", propertyFile.str());
properties.setown(createPTreeFromXMLFile(propertyFile.str()));
}
catch (IException *e)
{
EXCLOG(e, "Error processing properties file\n");
throwUnexpected();
}
// get the logfile specification
properties->getProp("@logDir", logDir.clear());
if (!logDir.length())
{
WARNLOG("logDir not specified in properties file - assuming code dir\n");
logDir.append(codeDir); //default to code dir
addPathSepChar(logDir);
logDir.append("logs"); //default folder name
}
recursiveCreateDirectory(logDir.str());
addPathSepChar(logDir);
rebuildLogfileName();
//get name of workunit job queue
StringBuffer sb;
properties->getProp("@name", sb.clear());
agentName.set(sb);
if (!agentName.length())
{
ERRLOG("'name' not specified in properties file\n");
throwUnexpected();
}
//get dali server(s)
properties->getProp("@daliServers", daliServers);
if (!daliServers.length())
{
ERRLOG("'daliServers' not specified in properties file\n");
throwUnexpected();
}
started = true;
Thread::start();
Thread::join();
}
virtual const void * nextGE(const void * seek, unsigned numFields) { throwUnexpected(); } // can only be called on stepping fields.
// IEngineRowStream
virtual void resetEOF() override { throwUnexpected(); }
virtual IEmbedServiceContext * createServiceContext(const char *service, unsigned flags, const char *options)
{
throwUnexpected();
return nullptr;
}
