void cancelReplicates(CActivityBase *activity, IPartDescriptor &partDesc)
{
RemoteFilename rfn;
IFileDescriptor &fileDesc = partDesc.queryOwner();
unsigned copies = partDesc.numCopies();
unsigned c=1;
for (; c<copies; c++)
{
unsigned replicateCopy;
unsigned clusterNum = partDesc.copyClusterNum(c, &replicateCopy);
rfn.clear();
partDesc.getFilename(c, rfn);
StringBuffer dstName;
rfn.getPath(dstName);
assertex(dstName.length());
if (replicateCopy>0)
{
try
{
queryThor().queryBackup().cancel(dstName.str());
}
catch (IException *e)
{
Owned<IThorException> re = MakeActivityException(activity, e, "Error cancelling backup '%s'", dstName.str());
ActPrintLog(&activity->queryContainer(), e, NULL);
e->Release();
}
}
}
}
//cloned from hthor - a candidate for commoning up.
static IKeyIndex *openKeyFile(IDistributedFilePart *keyFile)
{
unsigned numCopies = keyFile->numCopies();
assertex(numCopies);
for (unsigned copy=0; copy < numCopies; copy++)
{
RemoteFilename rfn;
try
{
OwnedIFile ifile = createIFile(keyFile->getFilename(rfn,copy));
unsigned __int64 thissize = ifile->size();
if (thissize != -1)
{
StringBuffer remotePath;
rfn.getRemotePath(remotePath);
unsigned crc = 0;
keyFile->getCrc(crc);
return createKeyIndex(remotePath.str(), crc, false, false);
}
}
catch (IException *E)
{
EXCLOG(E, "While opening index file");
E->Release();
}
}
RemoteFilename rfn;
StringBuffer url;
keyFile->getFilename(rfn).getRemotePath(url);
throw MakeStringException(1001, "Could not open key file at %s%s", url.str(), (numCopies > 1) ? " or any alternate location." : ".");
}
virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
{
isLocal = 0 != (TIWlocal & helper->getFlags());
mpTag = container.queryJobChannel().deserializeMPTag(data);
mpTag2 = container.queryJobChannel().deserializeMPTag(data);
data.read(active);
if (active)
{
data.read(logicalFilename);
partDesc.setown(deserializePartFileDescriptor(data));
}
data.read(singlePartKey);
data.read(refactor);
if (singlePartKey)
buildTlk = false;
else
{
data.read(buildTlk);
if (firstNode())
{
if (buildTlk)
tlkDesc.setown(deserializePartFileDescriptor(data));
else if (!isLocal) // existing tlk then..
{
tlkDesc.setown(deserializePartFileDescriptor(data));
unsigned c;
data.read(c);
while (c--)
{
RemoteFilename rf;
rf.deserialize(data);
if (!existingTlkIFile)
{
Owned<IFile> iFile = createIFile(rf);
if (iFile->exists())
existingTlkIFile.set(iFile);
}
}
if (!existingTlkIFile)
throw MakeActivityException(this, TE_FileNotFound, "Top level key part does not exist, for key");
}
}
}
IOutputMetaData * diskSize = helper->queryDiskRecordSize();
assertex(!(diskSize->getMetaFlags() & MDFneedserializedisk));
if (diskSize->isVariableSize())
{
if (TIWmaxlength & helper->getFlags())
maxDiskRecordSize = helper->getMaxKeySize();
else
maxDiskRecordSize = KEYBUILD_MAXLENGTH; //Current default behaviour, could be improved in the future
}
else
maxDiskRecordSize = diskSize->getFixedSize();
reportOverflow = false;
}
void setCanAccessDirectly(RemoteFilename & file,bool set)
{
if (set)
file.setPort(0);
else if (file.getPort()==0) // foreign daliservix may be passed in
file.setPort(getDaliServixPort());
}
void getDfuTempName(RemoteFilename & temp, const RemoteFilename & src)
{
StringBuffer ext;
src.split(NULL, NULL, NULL, &ext);
ext.append(".tmp");
temp.set(src);
temp.setExtension(ext);
}
static IKeyIndex *openKeyPart(CActivityBase *activity, const char *logicalFilename, IPartDescriptor &partDesc)
{
RemoteFilename rfn;
partDesc.getFilename(0, rfn);
StringBuffer filePath;
rfn.getPath(filePath);
unsigned crc=0;
partDesc.getCrc(crc);
Owned<IDelayedFile> lfile = queryThor().queryFileCache().lookup(*activity, partDesc);
return createKeyIndex(filePath.str(), crc, *lfile, false, false);
}
bool processPartitionCommand(ISocket * masterSocket, MemoryBuffer & msg, MemoryBuffer & results)
{
FileFormat srcFormat;
FileFormat tgtFormat;
unsigned whichInput;
RemoteFilename fullPath;
offset_t totalSize;
offset_t thisOffset;
offset_t thisSize;
unsigned thisHeaderSize;
unsigned numParts;
bool compressedInput = false;
unsigned compatflags = 0;
srcFormat.deserialize(msg);
tgtFormat.deserialize(msg);
msg.read(whichInput);
fullPath.deserialize(msg);
msg.read(totalSize);
msg.read(thisOffset);
msg.read(thisSize);
msg.read(thisHeaderSize);
msg.read(numParts);
if (msg.remaining())
msg.read(compressedInput);
if (msg.remaining())
msg.read(compatflags); // not yet used
StringAttr decryptkey;
if (msg.remaining())
msg.read(decryptkey);
if (msg.remaining())
{
srcFormat.deserializeExtra(msg, 1);
tgtFormat.deserializeExtra(msg, 1);
}
StringBuffer text;
fullPath.getRemotePath(text);
LOG(MCdebugProgress, unknownJob, "Process partition %d(%s)", whichInput, text.str());
Owned<IFormatProcessor> processor = createFormatProcessor(srcFormat, tgtFormat, true);
Owned<IOutputProcessor> target = createOutputProcessor(tgtFormat);
processor->setTarget(target);
processor->setPartitionRange(totalSize, thisOffset, thisSize, thisHeaderSize, numParts);
processor->setSource(whichInput, fullPath, compressedInput, decryptkey);
processor->calcPartitions(NULL);
PartitionPointArray partition;
processor->getResults(partition);
serialize(partition, results);
return true;
}
void CDiskReadSlaveActivityBase::kill()
{
if (!abortSoon && 0 != (helper->getFlags() & TDXtemporary) && !container.queryJob().queryUseCheckpoints())
{
if (1 == partDescs.ordinality() && !partDescs.item(0).queryOwner().queryProperties().getPropBool("@pausefile"))
{
IPartDescriptor &partDesc = partDescs.item(0);
RemoteFilename rfn;
partDesc.getFilename(0, rfn);
StringBuffer locationName;
rfn.getLocalPath(locationName);
container.queryTempHandler()->deregisterFile(locationName.str());
}
}
CSlaveActivity::kill();
}
static bool getFileInfo(RemoteFilename &fn, Owned<IFile> &f, offset_t &size,CDateTime &modtime)
{
f.setown(createIFile(fn));
bool isdir = false;
bool ret = f->getInfo(isdir,size,modtime);
if (ret&&isdir) {
StringBuffer fs;
fn.getRemotePath(fs);
throw MakeStringException(-1,"%s is a directory",fs.str());
}
return ret;
}
void renameDfuTempToFinal(const RemoteFilename & realname)
{
RemoteFilename tempFilename;
StringBuffer newTailname;
getDfuTempName(tempFilename, realname);
realname.getTail(newTailname);
OwnedIFile output = createIFile(tempFilename);
try
{
output->rename(newTailname);
}
catch (IException * e)
{
EXCLOG(e, "Failed to rename target file");
StringBuffer oldName;
realname.getPath(oldName);
LOG(MCdebugInfoDetail, unknownJob, "Error: Rename %s->%s failed - tring to delete target and rename again", oldName.str(), newTailname.str());
e->Release();
OwnedIFile old = createIFile(realname);
old->remove();
output->rename(newTailname);
}
}
void CWriteMasterBase::publish()
{
if (published) return;
published = true;
if (!(diskHelperBase->getFlags() & (TDXtemporary|TDXjobtemp)))
updateActivityResult(container.queryJob().queryWorkUnit(), diskHelperBase->getFlags(), diskHelperBase->getSequence(), fileName, recordsProcessed);
IPropertyTree &props = fileDesc->queryProperties();
props.setPropInt64("@recordCount", recordsProcessed);
if (0 == (diskHelperBase->getFlags() & TDXtemporary) || container.queryJob().queryUseCheckpoints())
{
if (0 != (diskHelperBase->getFlags() & TDWexpires))
setExpiryTime(props, diskHelperBase->getExpiryDays());
if (TDWupdate & diskHelperBase->getFlags())
{
unsigned eclCRC;
unsigned __int64 totalCRC;
diskHelperBase->getUpdateCRCs(eclCRC, totalCRC);
props.setPropInt("@eclCRC", eclCRC);
props.setPropInt64("@totalCRC", totalCRC);
}
}
container.queryTempHandler()->registerFile(fileName, container.queryOwner().queryGraphId(), diskHelperBase->getTempUsageCount(), TDXtemporary & diskHelperBase->getFlags(), getDiskOutputKind(diskHelperBase->getFlags()), &clusters);
if (!dlfn.isExternal())
{
bool temporary = 0 != (diskHelperBase->getFlags()&TDXtemporary);
if (!temporary && (queryJob().querySlaves() < fileDesc->numParts()))
{
// create empty parts for a fileDesc being published that is larger than this clusters
size32_t recordSize = 0;
IOutputMetaData *diskRowMeta = diskHelperBase->queryDiskRecordSize()->querySerializedDiskMeta();
if (diskRowMeta->isFixedSize() && (TAKdiskwrite == container.getKind()))
{
recordSize = diskRowMeta->getMinRecordSize();
if (0 != (diskHelperBase->getFlags() & TDXgrouped))
recordSize += 1;
}
unsigned compMethod = COMPRESS_METHOD_LZW;
// rowdiff used if recordSize > 0, else fallback to compMethod
if (getOptBool(THOROPT_COMP_FORCELZW, false))
{
recordSize = 0; // by default if fixed length (recordSize set), row diff compression is used. This forces compMethod.
compMethod = COMPRESS_METHOD_LZW;
}
else if (getOptBool(THOROPT_COMP_FORCEFLZ, false))
compMethod = COMPRESS_METHOD_FASTLZ;
else if (getOptBool(THOROPT_COMP_FORCELZ4, false))
compMethod = COMPRESS_METHOD_LZ4;
bool blockCompressed;
bool compressed = fileDesc->isCompressed(&blockCompressed);
for (unsigned clusterIdx=0; clusterIdx<fileDesc->numClusters(); clusterIdx++)
{
StringBuffer clusterName;
fileDesc->getClusterGroupName(clusterIdx, clusterName, &queryNamedGroupStore());
PROGLOG("Creating blank parts for file '%s', cluster '%s'", fileName.get(), clusterName.str());
unsigned p=0;
while (p<fileDesc->numParts())
{
if (p == targetOffset)
p += queryJob().querySlaves();
IPartDescriptor *partDesc = fileDesc->queryPart(p);
CDateTime createTime, modifiedTime;
for (unsigned c=0; c<partDesc->numCopies(); c++)
{
RemoteFilename rfn;
partDesc->getFilename(c, rfn);
StringBuffer path;
rfn.getPath(path);
try
{
ensureDirectoryForFile(path.str());
OwnedIFile iFile = createIFile(path.str());
Owned<IFileIO> iFileIO;
if (compressed)
iFileIO.setown(createCompressedFileWriter(iFile, recordSize, false, true, NULL, compMethod));
else
iFileIO.setown(iFile->open(IFOcreate));
dbgassertex(iFileIO.get());
iFileIO.clear();
// ensure copies have matching datestamps, as they would do normally (backupnode expects it)
if (partDesc->numCopies() > 1)
{
if (0 == c)
iFile->getTime(&createTime, &modifiedTime, NULL);
else
iFile->setTime(&createTime, &modifiedTime, NULL);
}
}
catch (IException *e)
{
if (0 == c)
throw;
Owned<IThorException> e2 = MakeThorException(e);
e->Release();
e2->setAction(tea_warning);
queryJob().fireException(e2);
}
}
partDesc->queryProperties().setPropInt64("@size", 0);
p++;
}
clusterIdx++;
}
}
queryThorFileManager().publish(container.queryJob(), fileName, *fileDesc, NULL);
}
}
static void test2()
{
const size32_t recsize = 17;
printf("Test DFS\n");
StringBuffer s;
unsigned i;
unsigned n;
unsigned t;
queryNamedGroupStore().remove("daregress_group");
queryDistributedFileDirectory().removeEntry("daregress::superfile1");
SocketEndpointArray epa;
for (n=0;n<400;n++) {
s.clear().append("192.168.").append(n/256).append('.').append(n%256);
SocketEndpoint ep(s.str());
epa.append(ep);
}
Owned<IGroup> group = createIGroup(epa);
queryNamedGroupStore().add("daregress_group",group,true);
if (!queryNamedGroupStore().find(group,s.clear()))
ERROR("Created logical group not found");
if (stricmp(s.str(),"daregress_group")!=0)
ERROR("Created logical group found with wrong name");
group.setown(queryNamedGroupStore().lookup("daregress_group"));
if (!group)
ERROR("named group lookup failed");
printf("Named group created - 400 nodes\n");
for (i=0;i<100;i++) {
Owned<IPropertyTree> pp = createPTree("Part");
Owned<IFileDescriptor>fdesc = createFileDescriptor();
fdesc->setDefaultDir("c:\\thordata\\regress");
n = 9;
for (unsigned k=0;k<400;k++) {
s.clear().append("192.168.").append(n/256).append('.').append(n%256);
Owned<INode> node = createINode(s.str());
pp->setPropInt64("@size",(n*777+i)*recsize);
s.clear().append("daregress_test").append(i).append("._").append(n+1).append("_of_400");
fdesc->setPart(n,node,s.str(),pp);
n = (n+9)%400;
}
fdesc->queryProperties().setPropInt("@recordSize",17);
s.clear().append("daregress::test").append(i);
queryDistributedFileDirectory().removeEntry(s.str());
StringBuffer cname;
Owned<IDistributedFile> dfile = queryDistributedFileDirectory().createNew(fdesc);
if (stricmp(dfile->getClusterName(0,cname),"daregress_group")!=0)
ERROR1("Cluster name wrong %d",i);
s.clear().append("daregress::test").append(i);
dfile->attach(s.str());
}
printf("DFile create done - 100 files\n");
unsigned samples = 5;
t = 33;
for (i=0;i<100;i++) {
s.clear().append("daregress::test").append(t);
if (!queryDistributedFileDirectory().exists(s.str()))
ERROR1("Could not find %s",s.str());
Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(s.str());
if (!dfile) {
ERROR1("Could not find %s",s.str());
continue;
}
offset_t totsz = 0;
n = 11;
for (unsigned k=0;k<400;k++) {
Owned<IDistributedFilePart> part = dfile->getPart(n);
if (!part) {
ERROR2("part not found %d %d",t,n);
continue;
}
s.clear().append("192.168.").append(n/256).append('.').append(n%256);
Owned<INode> node = createINode(s.str());
if (!node->equals(part->queryNode()))
ERROR2("part node mismatch %d, %d",t,n);
if (part->getFileSize(false,false)!=(n*777+t)*recsize)
ERROR4("size node mismatch %d, %d, %d, %d",t,n,(unsigned)part->getFileSize(false,false),(n*777+t)*recsize);
s.clear().append("daregress_test").append(t).append("._").append(n+1).append("_of_400");
/* ** TBD
if (stricmp(s.str(),part->queryPartName())!=0)
ERROR4("part name mismatch %d, %d '%s' '%s'",t,n,s.str(),part->queryPartName());
*/
totsz += (n*777+t)*recsize;
if ((samples>0)&&(i+n+t==k)) {
samples--;
RemoteFilename rfn;
part->getFilename(rfn,samples%2);
StringBuffer fn;
rfn.getRemotePath(fn);
printf("SAMPLE: %d,%d %s\n",t,n,fn.str());
}
n = (n+11)%400;
}
if (totsz!=dfile->getFileSize(false,false))
ERROR1("total size mismatch %d",t);
t = (t+33)%100;
}
printf("DFile lookup done - 100 files\n");
// check iteration
__int64 crctot = 0;
unsigned np = 0;
unsigned totrows = 0;
Owned<IDistributedFileIterator> fiter = queryDistributedFileDirectory().getIterator("daregress::*",false);
Owned<IDistributedFilePartIterator> piter;
ForEach(*fiter) {
piter.setown(fiter->query().getIterator());
ForEach(*piter) {
RemoteFilename rfn;
StringBuffer s;
piter->query().getFilename(rfn,0);
rfn.getRemotePath(s);
piter->query().getFilename(rfn,1);
rfn.getRemotePath(s);
crctot += crc32(s.str(),s.length(),0);
np++;
totrows += (unsigned)(piter->query().getFileSize(false,false)/fiter->query().queryProperties().getPropInt("@recordSize",-1));
}
}
piter.clear();
fiter.clear();
printf("DFile iterate done - %d parts, %d rows, CRC sum %"I64F"d\n",np,totrows,crctot);
Owned<IDistributedSuperFile> sfile;
sfile.setown(queryDistributedFileDirectory().createSuperFile("daregress::superfile1",true));
for (i = 0;i<100;i++) {
s.clear().append("daregress::test").append(i);
sfile->addSubFile(s.str());
}
sfile.clear();
sfile.setown(queryDistributedFileDirectory().lookupSuperFile("daregress::superfile1"));
if (!sfile) {
ERROR("Could not find added superfile");
return;
}
__int64 savcrc = crctot;
crctot = 0;
np = 0;
totrows = 0;
size32_t srs = (size32_t)sfile->queryProperties().getPropInt("@recordSize",-1);
if (srs!=17)
ERROR1("Superfile does not match subfile row size %d",srs);
piter.setown(sfile->getIterator());
ForEach(*piter) {
RemoteFilename rfn;
StringBuffer s;
piter->query().getFilename(rfn,0);
rfn.getRemotePath(s);
piter->query().getFilename(rfn,1);
rfn.getRemotePath(s);
crctot += crc32(s.str(),s.length(),0);
np++;
totrows += (unsigned)(piter->query().getFileSize(false,false)/srs);
}
piter.clear();
printf("Superfile iterate done - %d parts, %d rows, CRC sum %"I64F"d\n",np,totrows,crctot);
if (crctot!=savcrc)
ERROR("SuperFile does not match sub files");
unsigned tr = (unsigned)(sfile->getFileSize(false,false)/srs);
if (totrows!=tr)
ERROR1("Superfile size does not match part sum %d",tr);
sfile->detach();
sfile.clear();
sfile.setown(queryDistributedFileDirectory().lookupSuperFile("daregress::superfile1"));
if (sfile)
ERROR("Superfile deletion failed");
t = 37;
for (i=0;i<100;i++) {
s.clear().append("daregress::test").append(t);
if (i%1) {
Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(s.str());
if (!dfile)
ERROR1("Could not find %s",s.str());
dfile->detach();
}
else
queryDistributedFileDirectory().removeEntry(s.str());
t = (t+37)%100;
}
printf("DFile removal complete\n");
t = 39;
for (i=0;i<100;i++) {
if (queryDistributedFileDirectory().exists(s.str()))
ERROR1("Found %s after deletion",s.str());
Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(s.str());
if (dfile)
ERROR1("Found %s after deletion",s.str());
t = (t+39)%100;
}
printf("DFile removal check complete\n");
queryNamedGroupStore().remove("daregress_group");
if (queryNamedGroupStore().lookup("daregress_group"))
ERROR("Named group not removed");
}
void CFileSpraySoapBindingEx::downloadFile(IEspContext &context, CHttpRequest* request, CHttpResponse* response)
{
try
{
StringBuffer netAddressStr, osStr, pathStr, nameStr;
request->getParameter("NetAddress", netAddressStr);
request->getParameter("OS", osStr);
request->getParameter("Path", pathStr);
request->getParameter("Name", nameStr);
#if 0
StringArray files;
IProperties* params = request->queryParameters();
Owned<IPropertyIterator> iter = params->getIterator();
if (iter && iter->first())
{
while (iter->isValid())
{
const char *keyname=iter->getPropKey();
if (!keyname || strncmp(keyname, "Names", 5))
continue;
files.append(params->queryProp(iter->getPropKey()));
iter->next();
}
}
#endif
if (netAddressStr.length() < 1)
throw MakeStringException(ECLWATCH_INVALID_INPUT, "Network address not specified.");
if (pathStr.length() < 1)
throw MakeStringException(ECLWATCH_INVALID_INPUT, "Path not specified.");
if (nameStr.length() < 1)
throw MakeStringException(ECLWATCH_INVALID_INPUT,"File name not specified.");
char pathSep = '/';
if ((osStr.length() > 1) && (atoi(osStr.str())== OS_WINDOWS))
{
pathSep = '\\';
}
pathStr.replace(pathSep=='\\'?'/':'\\', pathSep);
if (*(pathStr.str() + pathStr.length() -1) != pathSep)
pathStr.append( pathSep );
StringBuffer fullName;
fullName.appendf("%s%s", pathStr.str(), nameStr.str());
StringBuffer headerStr("attachment;");
headerStr.appendf("filename=%s", nameStr.str());
RemoteFilename rfn;
rfn.setRemotePath(fullName.str());
SocketEndpoint ep(netAddressStr.str());
rfn.setIp(ep);
Owned<IFile> rFile = createIFile(rfn);
if (!rFile)
throw MakeStringException(ECLWATCH_CANNOT_OPEN_FILE,"Cannot open file %s.",fullName.str());
OwnedIFileIO rIO = rFile->openShared(IFOread,IFSHfull);
if (!rIO)
throw MakeStringException(ECLWATCH_CANNOT_READ_FILE,"Cannot read file %s.",fullName.str());
IFileIOStream* ioS = createIOStream(rIO);
context.addCustomerHeader("Content-disposition", headerStr.str());
response->setContent(ioS);
response->setContentType(HTTP_TYPE_OCTET_STREAM);
response->send();
}
catch(IException* e)
{
FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
}
return;
}
void prepareKey(IDistributedFile *index)
{
IDistributedFile *f = index;
IDistributedSuperFile *super = f->querySuperFile();
unsigned nparts = f->numParts(); // includes tlks if any, but unused in array
performPartLookup.ensure(nparts);
bool checkTLKConsistency = (nullptr != super) && !localKey && (0 != (TIRsorted & indexBaseHelper->getFlags()));
if (nofilter)
{
while (nparts--) performPartLookup.append(true);
if (!checkTLKConsistency) return;
}
else
{
while (nparts--) performPartLookup.append(false); // parts to perform lookup set later
}
Owned<IDistributedFileIterator> iter;
if (super)
{
iter.setown(super->getSubFileIterator(true));
verifyex(iter->first());
f = &iter->query();
}
unsigned width = f->numParts();
if (!localKey)
--width;
assertex(width);
unsigned tlkCrc = 0;
bool first = true;
unsigned superSubIndex=0;
bool fileCrc = false, rowCrc = false;
for (;;)
{
Owned<IDistributedFilePart> part = f->getPart(width);
if (checkTLKConsistency)
{
unsigned _tlkCrc;
if (part->getCrc(_tlkCrc))
fileCrc = true;
else if (part->queryAttributes().hasProp("@crc")) // NB: key "@crc" is not a crc on the file, but data within.
{
_tlkCrc = part->queryAttributes().getPropInt("@crc");
rowCrc = true;
}
else if (part->queryAttributes().hasProp("@tlkCrc")) // backward compat.
{
_tlkCrc = part->queryAttributes().getPropInt("@tlkCrc");
rowCrc = true;
}
else
{
if (rowCrc || fileCrc)
{
checkTLKConsistency = false;
Owned<IException> e = MakeActivityWarning(&container, 0, "Cannot validate that tlks in superfile %s match, some crc attributes are missing", super->queryLogicalName());
queryJobChannel().fireException(e);
}
}
if (rowCrc && fileCrc)
{
checkTLKConsistency = false;
Owned<IException> e = MakeActivityWarning(&container, 0, "Cannot validate that tlks in superfile %s match, due to mixed crc types.", super->queryLogicalName());
queryJobChannel().fireException(e);
}
if (checkTLKConsistency)
{
if (first)
{
tlkCrc = _tlkCrc;
first = false;
}
else if (tlkCrc != _tlkCrc)
throw MakeActivityException(this, 0, "Sorted output on super files comprising of non coparitioned sub keys is not supported (TLK's do not match)");
}
}
if (!nofilter)
{
Owned<IKeyIndex> keyIndex;
unsigned copy;
for (copy=0; copy<part->numCopies(); copy++)
{
RemoteFilename rfn;
OwnedIFile ifile = createIFile(part->getFilename(rfn,copy));
if (ifile->exists())
{
StringBuffer remotePath;
rfn.getRemotePath(remotePath);
unsigned crc = 0;
part->getCrc(crc);
keyIndex.setown(createKeyIndex(remotePath.str(), crc, false, false));
break;
}
}
if (!keyIndex)
throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", index->queryLogicalName());
unsigned fixedSize = indexBaseHelper->queryDiskRecordSize()->querySerializedDiskMeta()->getFixedSize(); // used only if fixed
Owned <IKeyManager> tlk = createLocalKeyManager(keyIndex, fixedSize, NULL);
indexBaseHelper->createSegmentMonitors(tlk);
tlk->finishSegmentMonitors();
tlk->reset();
while (tlk->lookup(false))
{
if (tlk->queryFpos())
performPartLookup.replace(true, (aindex_t)(super?super->numSubFiles(true)*(tlk->queryFpos()-1)+superSubIndex:tlk->queryFpos()-1));
}
}
if (!super||!iter->next())
break;
superSubIndex++;
f = &iter->query();
if (width != f->numParts()-1)
throw MakeActivityException(this, 0, "Super key %s, with mixture of sub key width are not supported.", f->queryLogicalName());
}
}
static bool physicalPartCopy(IFile *from,const char *tofile, Owned<IException> &exc, StringBuffer *tmpname)
{
StringBuffer tmpnamestr;
if (!tmpname)
tmpname = &tmpnamestr;
tmpname->append(tofile).append("__");
size32_t l = tmpname->length();
genUUID(*tmpname,true); // true for windows
StringAttr uuid(tmpname->str()+l);
tmpname->append(".tmp");
RemoteFilename tmpfn;
tmpfn.setRemotePath(tmpname->str());
//unsigned lastpc;
#ifdef LOG_PART_COPY
PROGLOG("start physicalPartCopy(%s,%s)",from->queryFilename(),tmpname->str());
#endif
try {
recursiveCreateDirectoryForFile(tmpname->str());
while(!asyncCopyFileSection(
uuid,
from,
tmpfn,
(offset_t)-1, // creates file
0,
(offset_t)-1, // all file
NULL,
PHYSICAL_COPY_POLL_TIME)) {
// Abort check TBD
}
}
catch (IException *e) {
EXCLOG(e,"SingleFileCopy: File copy error");
if (exc)
exc.setown(e);
else
e->Release();
}
Owned<IFile> f = createIFile(tmpfn);
if (!exc.get()&&(tmpnamestr.length()!=0)) {
try {
#ifdef LOG_PART_COPY
PROGLOG("physicalPartCopy rename(%s,%s)",tmpname->str(),pathTail(tofile));
#endif
f->rename(pathTail(tofile));
}
catch (IException *e) {
EXCLOG(e,"SingleFileCopy: File rename error");
if (exc)
exc.setown(e);
else
e->Release();
}
}
if (exc.get()) {
try {
f->remove();
}
catch (IException *e) {
// ignore
e->Release();
}
}
#ifdef LOG_PART_COPY
PROGLOG("done physicalPartCopy %s",(exc.get()==NULL)?"OK":"Failed");
#endif
return exc.get()==NULL;
}
virtual void init()
{
CMasterActivity::init();
OwnedRoxieString indexFileName(helper->getIndexFileName());
Owned<IDistributedFile> dataFile;
Owned<IDistributedFile> indexFile = queryThorFileManager().lookup(container.queryJob(), indexFileName, false, 0 != (helper->getJoinFlags() & JFindexoptional), true);
unsigned keyReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJKRR", 0);
if (!keyReadWidth || keyReadWidth>container.queryJob().querySlaves())
keyReadWidth = container.queryJob().querySlaves();
initMb.clear();
initMb.append(indexFileName.get());
if (helper->diskAccessRequired())
numTags += 2;
initMb.append(numTags);
unsigned t=0;
for (; t<numTags; t++)
{
tags[t] = container.queryJob().allocateMPTag();
initMb.append(tags[t]);
}
bool keyHasTlk = false;
if (indexFile)
{
unsigned numParts = 0;
localKey = indexFile->queryAttributes().getPropBool("@local");
if (container.queryLocalData() && !localKey)
throw MakeActivityException(this, 0, "Keyed Join cannot be LOCAL unless supplied index is local");
checkFormatCrc(this, indexFile, helper->getIndexFormatCrc(), true);
Owned<IFileDescriptor> indexFileDesc = indexFile->getFileDescriptor();
IDistributedSuperFile *superIndex = indexFile->querySuperFile();
unsigned superIndexWidth = 0;
unsigned numSuperIndexSubs = 0;
if (superIndex)
{
numSuperIndexSubs = superIndex->numSubFiles(true);
bool first=true;
// consistency check
Owned<IDistributedFileIterator> iter = superIndex->getSubFileIterator(true);
ForEach(*iter)
{
IDistributedFile &f = iter->query();
unsigned np = f.numParts()-1;
IDistributedFilePart &part = f.queryPart(np);
const char *kind = part.queryAttributes().queryProp("@kind");
bool hasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind); // if last part not tlk, then deemed local (might be singlePartKey)
if (first)
{
first = false;
keyHasTlk = hasTlk;
superIndexWidth = f.numParts();
if (keyHasTlk)
--superIndexWidth;
}
else
{
if (hasTlk != keyHasTlk)
throw MakeActivityException(this, 0, "Local/Single part keys cannot be mixed with distributed(tlk) keys in keyedjoin");
if (keyHasTlk && superIndexWidth != f.numParts()-1)
throw MakeActivityException(this, 0, "Super sub keys of different width cannot be mixed with distributed(tlk) keys in keyedjoin");
}
}
if (keyHasTlk)
numParts = superIndexWidth * numSuperIndexSubs;
else
numParts = superIndex->numParts();
}
else
{
numParts = indexFile->numParts();
if (numParts)
{
const char *kind = indexFile->queryPart(indexFile->numParts()-1).queryAttributes().queryProp("@kind");
keyHasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind);
if (keyHasTlk)
--numParts;
}
}
if (numParts)
{
initMb.append(numParts);
initMb.append(superIndexWidth); // 0 if not superIndex
initMb.append((superIndex && superIndex->isInterleaved()) ? numSuperIndexSubs : 0);
unsigned p=0;
UnsignedArray parts;
for (; p<numParts; p++)
parts.append(p);
indexFileDesc->serializeParts(initMb, parts);
if (localKey)
keyHasTlk = false; // not used
initMb.append(keyHasTlk);
if (keyHasTlk)
{
if (numSuperIndexSubs)
initMb.append(numSuperIndexSubs);
else
initMb.append((unsigned)1);
Owned<IDistributedFileIterator> iter;
IDistributedFile *f;
if (superIndex)
{
iter.setown(superIndex->getSubFileIterator(true));
f = &iter->query();
}
else
f = indexFile;
loop
{
unsigned location;
OwnedIFile iFile;
StringBuffer filePath;
Owned<IFileDescriptor> fileDesc = f->getFileDescriptor();
Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1);
if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath))
throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", f->queryLogicalName());
OwnedIFileIO iFileIO = iFile->open(IFOread);
assertex(iFileIO);
size32_t tlkSz = (size32_t)iFileIO->size();
initMb.append(tlkSz);
::read(iFileIO, 0, tlkSz, initMb);
if (!iter || !iter->next())
break;
f = &iter->query();
}
}
if (helper->diskAccessRequired())
{
OwnedRoxieString fetchFilename(helper->getFileName());
if (fetchFilename)
{
dataFile.setown(queryThorFileManager().lookup(container.queryJob(), fetchFilename, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true));
if (dataFile)
{
if (superIndex)
throw MakeActivityException(this, 0, "Superkeys and full keyed joins are not supported");
dataFileDesc.setown(getConfiguredFileDescriptor(*dataFile));
void *ekey;
size32_t ekeylen;
helper->getFileEncryptKey(ekeylen,ekey);
bool encrypted = dataFileDesc->queryProperties().getPropBool("@encrypted");
if (0 != ekeylen)
{
memset(ekey,0,ekeylen);
free(ekey);
if (!encrypted)
{
Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", dataFile->queryLogicalName());
container.queryJob().fireException(e);
}
}
else if (encrypted)
throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", dataFile->queryLogicalName());
/* If fetch file is local to cluster, fetches are sent to be processed to local node, each node has info about it's
* local parts only.
* If fetch file is off cluster, fetches are performed by requesting node directly on fetch part, therefore each nodes
* needs all part descriptors.
*/
remoteDataFiles = false;
RemoteFilename rfn;
dataFileDesc->queryPart(0)->getFilename(0, rfn);
if (!rfn.queryIP().ipequals(container.queryJob().querySlaveGroup().queryNode(0).endpoint()))
remoteDataFiles = true;
if (!remoteDataFiles) // local to cluster
{
unsigned dataReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJDRR", 0);
if (!dataReadWidth || dataReadWidth>container.queryJob().querySlaves())
dataReadWidth = container.queryJob().querySlaves();
Owned<IGroup> grp = container.queryJob().querySlaveGroup().subset((unsigned)0, dataReadWidth);
dataFileMapping.setown(getFileSlaveMaps(dataFile->queryLogicalName(), *dataFileDesc, container.queryJob().queryUserDescriptor(), *grp, false, false, NULL));
dataFileMapping->serializeFileOffsetMap(offsetMapMb.clear());
}
}
else
indexFile.clear();
}
}
}
else
bool canAccessDirectly(const RemoteFilename & file) // not that well named but historical
{
return (file.getPort()==0);
}
void verifyFile(const char *name,CDateTime *cutoff)
{
Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(name,UNKNOWN_USER);
if (!file)
return;
IPropertyTree &fileprops = file->queryAttributes();
bool blocked = false;
if (file->isCompressed(&blocked)&&!blocked)
return;
if (stopped)
return;
StringBuffer dtstr;
if (fileprops.getProp("@verified",dtstr)) {
if (!cutoff)
return;
CDateTime dt;
dt.setString(dtstr.str());
if (dt.compare(*cutoff)<=0)
return;
}
list.kill();
unsigned width = file->numParts();
unsigned short port = getDaliServixPort();
try {
Owned<IDistributedFilePart> testpart = file->getPart(0);
SocketEndpoint ep(testpart->queryNode()->endpoint());
if (!dafilesrvips.verifyDaliFileServer(ep)) {
StringBuffer ips;
ep.getIpText(ips);
PROGLOG("VERIFY: file %s, cannot run DAFILESRV on %s",name,ips.str());
return;
}
}
catch (IException *e)
{
StringBuffer s;
s.appendf("VERIFY: file %s",name);
EXCLOG(e, s.str());
e->Release();
return;
}
for (unsigned idx=0;idx<width;idx++) {
Owned<IDistributedFilePart> part = file->getPart(idx);
for (unsigned copy = 0; copy < part->numCopies(); copy++) {
if (stopped)
return;
unsigned reqcrc;
if (!part->getCrc(reqcrc))
continue;
SocketEndpoint ep(part->queryNode()->endpoint());
if (!dafilesrvips.verifyDaliFileServer(ep)) {
StringBuffer ips;
ep.getIpText(ips);
PROGLOG("VERIFY: file %s, cannot run DAFILESRV on %s",name,ips.str());
continue;
}
RemoteFilename rfn;
part->getFilename(rfn,copy);
rfn.setPort(port);
add(rfn,idx,copy,reqcrc);
}
}
if (list.ordinality()==0)
return;
PROGLOG("VERIFY: file %s started",name);
file.clear();
CriticalSection crit;
class casyncfor: public CAsyncFor
{
CFileCrcList *parent;
CriticalSection &crit;
public:
bool ok;
casyncfor(CFileCrcList *_parent, CriticalSection &_crit)
: crit(_crit)
{
parent = _parent;
ok = true;
}
void Do(unsigned i)
{
CriticalBlock block(crit);
if (parent->stopped)
return;
CFileCrcItem &item = parent->list.item(i);
RemoteFilename &rfn = item.filename;
Owned<IFile> partfile;
StringBuffer eps;
try
{
partfile.setown(createIFile(rfn));
// PROGLOG("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str());
if (partfile) {
if (parent->stopped)
return;
CriticalUnblock unblock(crit);
item.crc = partfile->getCRC();
}
else
ok = false;
}
catch (IException *e)
{
StringBuffer s;
s.appendf("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str());
EXCLOG(e, s.str());
e->Release();
ok = false;
}
}
} afor(this,crit);
afor.For(list.ordinality(),50,false,true);
ForEachItemIn(j,list) {
CFileCrcItem &item = list.item(j);
if (item.crc!=item.requiredcrc) {
if (stopped)
return;
StringBuffer rfs;
PROGLOG("VERIFY: FAILED %s (%x,%x) file %s",name,item.crc,item.requiredcrc,item.filename.getRemotePath(rfs).str());
afor.ok = false;
}
}
int main(int argc, char* argv[])
{
InitModuleObjects();
EnableSEHtoExceptionMapping();
if (argc<2) {
usage(argv[0]);
return -1;
}
bool forceGroupUpdate = false;
StringBuffer filename;
StringBuffer inst;
StringBuffer dcat;
StringBuffer dcomp;
StringBuffer dinst;
StringBuffer dip;
for (int i=1;i<argc;i++) {
if (argv[i][0]=='-') {
if ((stricmp(argv[i],"-i")==0)&&(i+1<argc)) {
inst.append(argv[++i]);
}
else if (0==stricmp(argv[i],"-f")) {
forceGroupUpdate = true;
}
else if ((stricmp(argv[i],"-d")==0)&&(i+3<argc)) {
dcat.append(argv[++i]);
dcomp.append(argv[++i]);
dinst.append(argv[++i]);
}
else if ((stricmp(argv[i],"-ip")==0)&&(i+1<argc)) {
dip.append(argv[++i]);
}
else {
usage(argv[0]);
return -1;
}
}
else {
if (filename.length()) {
usage(argv[0]);
return -1;
}
filename.append(argv[i]);
}
}
Owned<IPropertyTree> env;
try {
env.setown(createPTreeFromXMLFile(argv[1]));
if (!env.get()) {
fprintf(stderr,"Could not load Environment from %s\n",argv[1]);
return 1;
}
const char *s = env->queryName();
if (!s||(strcmp(s,"Environment")!=0)) {
fprintf(stderr,"File %s is invalid\n",argv[1]);
return 1;
}
}
catch (IException *e) {
StringBuffer err;
e->errorMessage(err);
fprintf(stderr,"Could not load Environment from %s, %s\n",argv[1],err.str());
return 1;
}
int ret = 0;
try {
if (dcat.length()) {
IPropertyTree* dirs = env->queryPropTree("Software/Directories");
StringBuffer dirout;
if (getConfigurationDirectory(dirs,dcat.str(),dcomp.str(),dinst.str(),dirout)&&dirout.length()) {
if (dip.length()) {
SocketEndpoint ep(dip.str());
RemoteFilename rfn;
rfn.setPath(ep,dirout.str());
rfn.getRemotePath(dirout.clear());
}
printf("%s",dirout.str());
}
else {
ret = 1;
}
}
else {
if (!updateDaliEnv(env, forceGroupUpdate, inst.str()))
ret = 1;
}
}
catch (IException *e) {
pexception("updtdalienv",e);
e->Release();
ret = 1;
}
releaseAtoms();
return ret;
}
void CachedPasswordProvider::addPasswordForFilename(RemoteFilename & filename)
{
addPasswordForIp(filename.queryIP());
}
void CachedPasswordProvider::addPasswordForFilename(const char * filename)
{
RemoteFilename remote;
remote.setRemotePath(filename);
addPasswordForIp(remote.queryIP());
}
static void _doReplicate(CActivityBase *activity, IPartDescriptor &partDesc, ICopyFileProgress *iProgress)
{
StringBuffer primaryName;
getPartFilename(partDesc, 0, primaryName);;
RemoteFilename rfn;
IFileDescriptor &fileDesc = partDesc.queryOwner();
unsigned copies = partDesc.numCopies();
unsigned c=1;
for (; c<copies; c++)
{
unsigned replicateCopy;
unsigned clusterNum = partDesc.copyClusterNum(c, &replicateCopy);
rfn.clear();
partDesc.getFilename(c, rfn);
StringBuffer dstName;
rfn.getPath(dstName);
assertex(dstName.length());
if (replicateCopy>0 )
{
try
{
queryThor().queryBackup().backup(dstName.str(), primaryName.str());
}
catch (IException *e)
{
Owned<IThorException> re = MakeActivityWarning(activity, e, "Failed to create replicate file '%s'", dstName.str());
e->Release();
activity->fireException(re);
}
}
else // another primary
{
ActPrintLog(activity, "Copying to primary %s", dstName.str());
StringBuffer tmpName(dstName.str());
tmpName.append(".tmp");
OwnedIFile tmpIFile = createIFile(tmpName.str());
OwnedIFile srcFile = createIFile(primaryName.str());
CFIPScope fipScope(tmpName.str());
try
{
try
{
ensureDirectoryForFile(dstName.str());
::copyFile(tmpIFile, srcFile, 0x100000, iProgress);
}
catch (IException *e)
{
IThorException *re = MakeActivityException(activity, e, "Failed to copy to tmp file '%s' from source file '%s'", tmpIFile->queryFilename(), srcFile->queryFilename());
e->Release();
throw re;
}
try
{
OwnedIFile dstIFile = createIFile(dstName.str());
dstIFile->remove();
tmpIFile->rename(pathTail(dstName.str()));
}
catch (IException *e)
{
IThorException *re = ThorWrapException(e, "Failed to rename '%s' to '%s'", tmpName.str(), dstName.str());
e->Release();
throw re;
}
}
catch (IException *)
{
try { tmpIFile->remove(); }
catch (IException *e) { ActPrintLog(&activity->queryContainer(), e, NULL); e->Release(); }
throw;
}
}
}
}
