//cloned from hthor - a candidate for commoning up.
static IKeyIndex *openKeyFile(IDistributedFilePart *keyFile)
{
unsigned numCopies = keyFile->numCopies();
assertex(numCopies);
for (unsigned copy=0; copy < numCopies; copy++)
{
RemoteFilename rfn;
try
{
OwnedIFile ifile = createIFile(keyFile->getFilename(rfn,copy));
unsigned __int64 thissize = ifile->size();
if (thissize != -1)
{
StringBuffer remotePath;
rfn.getRemotePath(remotePath);
unsigned crc = 0;
keyFile->getCrc(crc);
return createKeyIndex(remotePath.str(), crc, false, false);
}
}
catch (IException *E)
{
EXCLOG(E, "While opening index file");
E->Release();
}
}
RemoteFilename rfn;
StringBuffer url;
keyFile->getFilename(rfn).getRemotePath(url);
throw MakeStringException(1001, "Could not open key file at %s%s", url.str(), (numCopies > 1) ? " or any alternate location." : ".");
}
bool processPartitionCommand(ISocket * masterSocket, MemoryBuffer & msg, MemoryBuffer & results)
{
FileFormat srcFormat;
FileFormat tgtFormat;
unsigned whichInput;
RemoteFilename fullPath;
offset_t totalSize;
offset_t thisOffset;
offset_t thisSize;
unsigned thisHeaderSize;
unsigned numParts;
bool compressedInput = false;
unsigned compatflags = 0;
srcFormat.deserialize(msg);
tgtFormat.deserialize(msg);
msg.read(whichInput);
fullPath.deserialize(msg);
msg.read(totalSize);
msg.read(thisOffset);
msg.read(thisSize);
msg.read(thisHeaderSize);
msg.read(numParts);
if (msg.remaining())
msg.read(compressedInput);
if (msg.remaining())
msg.read(compatflags); // not yet used
StringAttr decryptkey;
if (msg.remaining())
msg.read(decryptkey);
if (msg.remaining())
{
srcFormat.deserializeExtra(msg, 1);
tgtFormat.deserializeExtra(msg, 1);
}
StringBuffer text;
fullPath.getRemotePath(text);
LOG(MCdebugProgress, unknownJob, "Process partition %d(%s)", whichInput, text.str());
Owned<IFormatProcessor> processor = createFormatProcessor(srcFormat, tgtFormat, true);
Owned<IOutputProcessor> target = createOutputProcessor(tgtFormat);
processor->setTarget(target);
processor->setPartitionRange(totalSize, thisOffset, thisSize, thisHeaderSize, numParts);
processor->setSource(whichInput, fullPath, compressedInput, decryptkey);
processor->calcPartitions(NULL);
PartitionPointArray partition;
processor->getResults(partition);
serialize(partition, results);
return true;
}
static bool getFileInfo(RemoteFilename &fn, Owned<IFile> &f, offset_t &size,CDateTime &modtime)
{
f.setown(createIFile(fn));
bool isdir = false;
bool ret = f->getInfo(isdir,size,modtime);
if (ret&&isdir) {
StringBuffer fs;
fn.getRemotePath(fs);
throw MakeStringException(-1,"%s is a directory",fs.str());
}
return ret;
}
static void test2()
{
const size32_t recsize = 17;
printf("Test DFS\n");
StringBuffer s;
unsigned i;
unsigned n;
unsigned t;
queryNamedGroupStore().remove("daregress_group");
queryDistributedFileDirectory().removeEntry("daregress::superfile1");
SocketEndpointArray epa;
for (n=0;n<400;n++) {
s.clear().append("192.168.").append(n/256).append('.').append(n%256);
SocketEndpoint ep(s.str());
epa.append(ep);
}
Owned<IGroup> group = createIGroup(epa);
queryNamedGroupStore().add("daregress_group",group,true);
if (!queryNamedGroupStore().find(group,s.clear()))
ERROR("Created logical group not found");
if (stricmp(s.str(),"daregress_group")!=0)
ERROR("Created logical group found with wrong name");
group.setown(queryNamedGroupStore().lookup("daregress_group"));
if (!group)
ERROR("named group lookup failed");
printf("Named group created - 400 nodes\n");
for (i=0;i<100;i++) {
Owned<IPropertyTree> pp = createPTree("Part");
Owned<IFileDescriptor>fdesc = createFileDescriptor();
fdesc->setDefaultDir("c:\\thordata\\regress");
n = 9;
for (unsigned k=0;k<400;k++) {
s.clear().append("192.168.").append(n/256).append('.').append(n%256);
Owned<INode> node = createINode(s.str());
pp->setPropInt64("@size",(n*777+i)*recsize);
s.clear().append("daregress_test").append(i).append("._").append(n+1).append("_of_400");
fdesc->setPart(n,node,s.str(),pp);
n = (n+9)%400;
}
fdesc->queryProperties().setPropInt("@recordSize",17);
s.clear().append("daregress::test").append(i);
queryDistributedFileDirectory().removeEntry(s.str());
StringBuffer cname;
Owned<IDistributedFile> dfile = queryDistributedFileDirectory().createNew(fdesc);
if (stricmp(dfile->getClusterName(0,cname),"daregress_group")!=0)
ERROR1("Cluster name wrong %d",i);
s.clear().append("daregress::test").append(i);
dfile->attach(s.str());
}
printf("DFile create done - 100 files\n");
unsigned samples = 5;
t = 33;
for (i=0;i<100;i++) {
s.clear().append("daregress::test").append(t);
if (!queryDistributedFileDirectory().exists(s.str()))
ERROR1("Could not find %s",s.str());
Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(s.str());
if (!dfile) {
ERROR1("Could not find %s",s.str());
continue;
}
offset_t totsz = 0;
n = 11;
for (unsigned k=0;k<400;k++) {
Owned<IDistributedFilePart> part = dfile->getPart(n);
if (!part) {
ERROR2("part not found %d %d",t,n);
continue;
}
s.clear().append("192.168.").append(n/256).append('.').append(n%256);
Owned<INode> node = createINode(s.str());
if (!node->equals(part->queryNode()))
ERROR2("part node mismatch %d, %d",t,n);
if (part->getFileSize(false,false)!=(n*777+t)*recsize)
ERROR4("size node mismatch %d, %d, %d, %d",t,n,(unsigned)part->getFileSize(false,false),(n*777+t)*recsize);
s.clear().append("daregress_test").append(t).append("._").append(n+1).append("_of_400");
/* ** TBD
if (stricmp(s.str(),part->queryPartName())!=0)
ERROR4("part name mismatch %d, %d '%s' '%s'",t,n,s.str(),part->queryPartName());
*/
totsz += (n*777+t)*recsize;
if ((samples>0)&&(i+n+t==k)) {
samples--;
RemoteFilename rfn;
part->getFilename(rfn,samples%2);
StringBuffer fn;
rfn.getRemotePath(fn);
printf("SAMPLE: %d,%d %s\n",t,n,fn.str());
}
n = (n+11)%400;
}
if (totsz!=dfile->getFileSize(false,false))
ERROR1("total size mismatch %d",t);
t = (t+33)%100;
}
printf("DFile lookup done - 100 files\n");
// check iteration
__int64 crctot = 0;
unsigned np = 0;
unsigned totrows = 0;
Owned<IDistributedFileIterator> fiter = queryDistributedFileDirectory().getIterator("daregress::*",false);
Owned<IDistributedFilePartIterator> piter;
ForEach(*fiter) {
piter.setown(fiter->query().getIterator());
ForEach(*piter) {
RemoteFilename rfn;
StringBuffer s;
piter->query().getFilename(rfn,0);
rfn.getRemotePath(s);
piter->query().getFilename(rfn,1);
rfn.getRemotePath(s);
crctot += crc32(s.str(),s.length(),0);
np++;
totrows += (unsigned)(piter->query().getFileSize(false,false)/fiter->query().queryProperties().getPropInt("@recordSize",-1));
}
}
piter.clear();
fiter.clear();
printf("DFile iterate done - %d parts, %d rows, CRC sum %"I64F"d\n",np,totrows,crctot);
Owned<IDistributedSuperFile> sfile;
sfile.setown(queryDistributedFileDirectory().createSuperFile("daregress::superfile1",true));
for (i = 0;i<100;i++) {
s.clear().append("daregress::test").append(i);
sfile->addSubFile(s.str());
}
sfile.clear();
sfile.setown(queryDistributedFileDirectory().lookupSuperFile("daregress::superfile1"));
if (!sfile) {
ERROR("Could not find added superfile");
return;
}
__int64 savcrc = crctot;
crctot = 0;
np = 0;
totrows = 0;
size32_t srs = (size32_t)sfile->queryProperties().getPropInt("@recordSize",-1);
if (srs!=17)
ERROR1("Superfile does not match subfile row size %d",srs);
piter.setown(sfile->getIterator());
ForEach(*piter) {
RemoteFilename rfn;
StringBuffer s;
piter->query().getFilename(rfn,0);
rfn.getRemotePath(s);
piter->query().getFilename(rfn,1);
rfn.getRemotePath(s);
crctot += crc32(s.str(),s.length(),0);
np++;
totrows += (unsigned)(piter->query().getFileSize(false,false)/srs);
}
piter.clear();
printf("Superfile iterate done - %d parts, %d rows, CRC sum %"I64F"d\n",np,totrows,crctot);
if (crctot!=savcrc)
ERROR("SuperFile does not match sub files");
unsigned tr = (unsigned)(sfile->getFileSize(false,false)/srs);
if (totrows!=tr)
ERROR1("Superfile size does not match part sum %d",tr);
sfile->detach();
sfile.clear();
sfile.setown(queryDistributedFileDirectory().lookupSuperFile("daregress::superfile1"));
if (sfile)
ERROR("Superfile deletion failed");
t = 37;
for (i=0;i<100;i++) {
s.clear().append("daregress::test").append(t);
if (i%1) {
Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(s.str());
if (!dfile)
ERROR1("Could not find %s",s.str());
dfile->detach();
}
else
queryDistributedFileDirectory().removeEntry(s.str());
t = (t+37)%100;
}
printf("DFile removal complete\n");
t = 39;
for (i=0;i<100;i++) {
if (queryDistributedFileDirectory().exists(s.str()))
ERROR1("Found %s after deletion",s.str());
Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(s.str());
if (dfile)
ERROR1("Found %s after deletion",s.str());
t = (t+39)%100;
}
printf("DFile removal check complete\n");
queryNamedGroupStore().remove("daregress_group");
if (queryNamedGroupStore().lookup("daregress_group"))
ERROR("Named group not removed");
}
void prepareKey(IDistributedFile *index)
{
IDistributedFile *f = index;
IDistributedSuperFile *super = f->querySuperFile();
unsigned nparts = f->numParts(); // includes tlks if any, but unused in array
performPartLookup.ensure(nparts);
bool checkTLKConsistency = (nullptr != super) && !localKey && (0 != (TIRsorted & indexBaseHelper->getFlags()));
if (nofilter)
{
while (nparts--) performPartLookup.append(true);
if (!checkTLKConsistency) return;
}
else
{
while (nparts--) performPartLookup.append(false); // parts to perform lookup set later
}
Owned<IDistributedFileIterator> iter;
if (super)
{
iter.setown(super->getSubFileIterator(true));
verifyex(iter->first());
f = &iter->query();
}
unsigned width = f->numParts();
if (!localKey)
--width;
assertex(width);
unsigned tlkCrc = 0;
bool first = true;
unsigned superSubIndex=0;
bool fileCrc = false, rowCrc = false;
for (;;)
{
Owned<IDistributedFilePart> part = f->getPart(width);
if (checkTLKConsistency)
{
unsigned _tlkCrc;
if (part->getCrc(_tlkCrc))
fileCrc = true;
else if (part->queryAttributes().hasProp("@crc")) // NB: key "@crc" is not a crc on the file, but data within.
{
_tlkCrc = part->queryAttributes().getPropInt("@crc");
rowCrc = true;
}
else if (part->queryAttributes().hasProp("@tlkCrc")) // backward compat.
{
_tlkCrc = part->queryAttributes().getPropInt("@tlkCrc");
rowCrc = true;
}
else
{
if (rowCrc || fileCrc)
{
checkTLKConsistency = false;
Owned<IException> e = MakeActivityWarning(&container, 0, "Cannot validate that tlks in superfile %s match, some crc attributes are missing", super->queryLogicalName());
queryJobChannel().fireException(e);
}
}
if (rowCrc && fileCrc)
{
checkTLKConsistency = false;
Owned<IException> e = MakeActivityWarning(&container, 0, "Cannot validate that tlks in superfile %s match, due to mixed crc types.", super->queryLogicalName());
queryJobChannel().fireException(e);
}
if (checkTLKConsistency)
{
if (first)
{
tlkCrc = _tlkCrc;
first = false;
}
else if (tlkCrc != _tlkCrc)
throw MakeActivityException(this, 0, "Sorted output on super files comprising of non coparitioned sub keys is not supported (TLK's do not match)");
}
}
if (!nofilter)
{
Owned<IKeyIndex> keyIndex;
unsigned copy;
for (copy=0; copy<part->numCopies(); copy++)
{
RemoteFilename rfn;
OwnedIFile ifile = createIFile(part->getFilename(rfn,copy));
if (ifile->exists())
{
StringBuffer remotePath;
rfn.getRemotePath(remotePath);
unsigned crc = 0;
part->getCrc(crc);
keyIndex.setown(createKeyIndex(remotePath.str(), crc, false, false));
break;
}
}
if (!keyIndex)
throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", index->queryLogicalName());
unsigned fixedSize = indexBaseHelper->queryDiskRecordSize()->querySerializedDiskMeta()->getFixedSize(); // used only if fixed
Owned <IKeyManager> tlk = createLocalKeyManager(keyIndex, fixedSize, NULL);
indexBaseHelper->createSegmentMonitors(tlk);
tlk->finishSegmentMonitors();
tlk->reset();
while (tlk->lookup(false))
{
if (tlk->queryFpos())
performPartLookup.replace(true, (aindex_t)(super?super->numSubFiles(true)*(tlk->queryFpos()-1)+superSubIndex:tlk->queryFpos()-1));
}
}
if (!super||!iter->next())
break;
superSubIndex++;
f = &iter->query();
if (width != f->numParts()-1)
throw MakeActivityException(this, 0, "Super key %s, with mixture of sub key width are not supported.", f->queryLogicalName());
}
}
int main(int argc, char* argv[])
{
InitModuleObjects();
EnableSEHtoExceptionMapping();
if (argc<2) {
usage(argv[0]);
return -1;
}
bool forceGroupUpdate = false;
StringBuffer filename;
StringBuffer inst;
StringBuffer dcat;
StringBuffer dcomp;
StringBuffer dinst;
StringBuffer dip;
for (int i=1;i<argc;i++) {
if (argv[i][0]=='-') {
if ((stricmp(argv[i],"-i")==0)&&(i+1<argc)) {
inst.append(argv[++i]);
}
else if (0==stricmp(argv[i],"-f")) {
forceGroupUpdate = true;
}
else if ((stricmp(argv[i],"-d")==0)&&(i+3<argc)) {
dcat.append(argv[++i]);
dcomp.append(argv[++i]);
dinst.append(argv[++i]);
}
else if ((stricmp(argv[i],"-ip")==0)&&(i+1<argc)) {
dip.append(argv[++i]);
}
else {
usage(argv[0]);
return -1;
}
}
else {
if (filename.length()) {
usage(argv[0]);
return -1;
}
filename.append(argv[i]);
}
}
Owned<IPropertyTree> env;
try {
env.setown(createPTreeFromXMLFile(argv[1]));
if (!env.get()) {
fprintf(stderr,"Could not load Environment from %s\n",argv[1]);
return 1;
}
const char *s = env->queryName();
if (!s||(strcmp(s,"Environment")!=0)) {
fprintf(stderr,"File %s is invalid\n",argv[1]);
return 1;
}
}
catch (IException *e) {
StringBuffer err;
e->errorMessage(err);
fprintf(stderr,"Could not load Environment from %s, %s\n",argv[1],err.str());
return 1;
}
int ret = 0;
try {
if (dcat.length()) {
IPropertyTree* dirs = env->queryPropTree("Software/Directories");
StringBuffer dirout;
if (getConfigurationDirectory(dirs,dcat.str(),dcomp.str(),dinst.str(),dirout)&&dirout.length()) {
if (dip.length()) {
SocketEndpoint ep(dip.str());
RemoteFilename rfn;
rfn.setPath(ep,dirout.str());
rfn.getRemotePath(dirout.clear());
}
printf("%s",dirout.str());
}
else {
ret = 1;
}
}
else {
if (!updateDaliEnv(env, forceGroupUpdate, inst.str()))
ret = 1;
}
}
catch (IException *e) {
pexception("updtdalienv",e);
e->Release();
ret = 1;
}
releaseAtoms();
return ret;
}
