//cloned from hthor - a candidate for commoning up. static IKeyIndex *openKeyFile(IDistributedFilePart *keyFile) { unsigned numCopies = keyFile->numCopies(); assertex(numCopies); for (unsigned copy=0; copy < numCopies; copy++) { RemoteFilename rfn; try { OwnedIFile ifile = createIFile(keyFile->getFilename(rfn,copy)); unsigned __int64 thissize = ifile->size(); if (thissize != -1) { StringBuffer remotePath; rfn.getRemotePath(remotePath); unsigned crc = 0; keyFile->getCrc(crc); return createKeyIndex(remotePath.str(), crc, false, false); } } catch (IException *E) { EXCLOG(E, "While opening index file"); E->Release(); } } RemoteFilename rfn; StringBuffer url; keyFile->getFilename(rfn).getRemotePath(url); throw MakeStringException(1001, "Could not open key file at %s%s", url.str(), (numCopies > 1) ? " or any alternate location." : "."); }
bool processPartitionCommand(ISocket * masterSocket, MemoryBuffer & msg, MemoryBuffer & results) { FileFormat srcFormat; FileFormat tgtFormat; unsigned whichInput; RemoteFilename fullPath; offset_t totalSize; offset_t thisOffset; offset_t thisSize; unsigned thisHeaderSize; unsigned numParts; bool compressedInput = false; unsigned compatflags = 0; srcFormat.deserialize(msg); tgtFormat.deserialize(msg); msg.read(whichInput); fullPath.deserialize(msg); msg.read(totalSize); msg.read(thisOffset); msg.read(thisSize); msg.read(thisHeaderSize); msg.read(numParts); if (msg.remaining()) msg.read(compressedInput); if (msg.remaining()) msg.read(compatflags); // not yet used StringAttr decryptkey; if (msg.remaining()) msg.read(decryptkey); if (msg.remaining()) { srcFormat.deserializeExtra(msg, 1); tgtFormat.deserializeExtra(msg, 1); } StringBuffer text; fullPath.getRemotePath(text); LOG(MCdebugProgress, unknownJob, "Process partition %d(%s)", whichInput, text.str()); Owned<IFormatProcessor> processor = createFormatProcessor(srcFormat, tgtFormat, true); Owned<IOutputProcessor> target = createOutputProcessor(tgtFormat); processor->setTarget(target); processor->setPartitionRange(totalSize, thisOffset, thisSize, thisHeaderSize, numParts); processor->setSource(whichInput, fullPath, compressedInput, decryptkey); processor->calcPartitions(NULL); PartitionPointArray partition; processor->getResults(partition); serialize(partition, results); return true; }
static bool getFileInfo(RemoteFilename &fn, Owned<IFile> &f, offset_t &size,CDateTime &modtime) { f.setown(createIFile(fn)); bool isdir = false; bool ret = f->getInfo(isdir,size,modtime); if (ret&&isdir) { StringBuffer fs; fn.getRemotePath(fs); throw MakeStringException(-1,"%s is a directory",fs.str()); } return ret; }
static void test2() { const size32_t recsize = 17; printf("Test DFS\n"); StringBuffer s; unsigned i; unsigned n; unsigned t; queryNamedGroupStore().remove("daregress_group"); queryDistributedFileDirectory().removeEntry("daregress::superfile1"); SocketEndpointArray epa; for (n=0;n<400;n++) { s.clear().append("192.168.").append(n/256).append('.').append(n%256); SocketEndpoint ep(s.str()); epa.append(ep); } Owned<IGroup> group = createIGroup(epa); queryNamedGroupStore().add("daregress_group",group,true); if (!queryNamedGroupStore().find(group,s.clear())) ERROR("Created logical group not found"); if (stricmp(s.str(),"daregress_group")!=0) ERROR("Created logical group found with wrong name"); group.setown(queryNamedGroupStore().lookup("daregress_group")); if (!group) ERROR("named group lookup failed"); printf("Named group created - 400 nodes\n"); for (i=0;i<100;i++) { Owned<IPropertyTree> pp = createPTree("Part"); Owned<IFileDescriptor>fdesc = createFileDescriptor(); fdesc->setDefaultDir("c:\\thordata\\regress"); n = 9; for (unsigned k=0;k<400;k++) { s.clear().append("192.168.").append(n/256).append('.').append(n%256); Owned<INode> node = createINode(s.str()); pp->setPropInt64("@size",(n*777+i)*recsize); s.clear().append("daregress_test").append(i).append("._").append(n+1).append("_of_400"); fdesc->setPart(n,node,s.str(),pp); n = (n+9)%400; } fdesc->queryProperties().setPropInt("@recordSize",17); s.clear().append("daregress::test").append(i); queryDistributedFileDirectory().removeEntry(s.str()); StringBuffer cname; Owned<IDistributedFile> dfile = queryDistributedFileDirectory().createNew(fdesc); if (stricmp(dfile->getClusterName(0,cname),"daregress_group")!=0) ERROR1("Cluster name wrong %d",i); s.clear().append("daregress::test").append(i); dfile->attach(s.str()); } printf("DFile create done - 100 files\n"); unsigned samples = 5; t = 33; for (i=0;i<100;i++) { s.clear().append("daregress::test").append(t); if (!queryDistributedFileDirectory().exists(s.str())) ERROR1("Could not find %s",s.str()); Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(s.str()); if (!dfile) { ERROR1("Could not find %s",s.str()); continue; } offset_t totsz = 0; n = 11; for (unsigned k=0;k<400;k++) { Owned<IDistributedFilePart> part = dfile->getPart(n); if (!part) { ERROR2("part not found %d %d",t,n); continue; } s.clear().append("192.168.").append(n/256).append('.').append(n%256); Owned<INode> node = createINode(s.str()); if (!node->equals(part->queryNode())) ERROR2("part node mismatch %d, %d",t,n); if (part->getFileSize(false,false)!=(n*777+t)*recsize) ERROR4("size node mismatch %d, %d, %d, %d",t,n,(unsigned)part->getFileSize(false,false),(n*777+t)*recsize); s.clear().append("daregress_test").append(t).append("._").append(n+1).append("_of_400"); /* ** TBD if (stricmp(s.str(),part->queryPartName())!=0) ERROR4("part name mismatch %d, %d '%s' '%s'",t,n,s.str(),part->queryPartName()); */ totsz += (n*777+t)*recsize; if ((samples>0)&&(i+n+t==k)) { samples--; RemoteFilename rfn; part->getFilename(rfn,samples%2); StringBuffer fn; rfn.getRemotePath(fn); printf("SAMPLE: %d,%d %s\n",t,n,fn.str()); } n = (n+11)%400; } if (totsz!=dfile->getFileSize(false,false)) ERROR1("total size mismatch %d",t); t = (t+33)%100; } printf("DFile lookup done - 100 files\n"); // check iteration __int64 crctot = 0; unsigned np = 0; unsigned totrows = 0; Owned<IDistributedFileIterator> fiter = queryDistributedFileDirectory().getIterator("daregress::*",false); Owned<IDistributedFilePartIterator> piter; ForEach(*fiter) { piter.setown(fiter->query().getIterator()); ForEach(*piter) { RemoteFilename rfn; StringBuffer s; piter->query().getFilename(rfn,0); rfn.getRemotePath(s); piter->query().getFilename(rfn,1); rfn.getRemotePath(s); crctot += crc32(s.str(),s.length(),0); np++; totrows += (unsigned)(piter->query().getFileSize(false,false)/fiter->query().queryProperties().getPropInt("@recordSize",-1)); } } piter.clear(); fiter.clear(); printf("DFile iterate done - %d parts, %d rows, CRC sum %"I64F"d\n",np,totrows,crctot); Owned<IDistributedSuperFile> sfile; sfile.setown(queryDistributedFileDirectory().createSuperFile("daregress::superfile1",true)); for (i = 0;i<100;i++) { s.clear().append("daregress::test").append(i); sfile->addSubFile(s.str()); } sfile.clear(); sfile.setown(queryDistributedFileDirectory().lookupSuperFile("daregress::superfile1")); if (!sfile) { ERROR("Could not find added superfile"); return; } __int64 savcrc = crctot; crctot = 0; np = 0; totrows = 0; size32_t srs = (size32_t)sfile->queryProperties().getPropInt("@recordSize",-1); if (srs!=17) ERROR1("Superfile does not match subfile row size %d",srs); piter.setown(sfile->getIterator()); ForEach(*piter) { RemoteFilename rfn; StringBuffer s; piter->query().getFilename(rfn,0); rfn.getRemotePath(s); piter->query().getFilename(rfn,1); rfn.getRemotePath(s); crctot += crc32(s.str(),s.length(),0); np++; totrows += (unsigned)(piter->query().getFileSize(false,false)/srs); } piter.clear(); printf("Superfile iterate done - %d parts, %d rows, CRC sum %"I64F"d\n",np,totrows,crctot); if (crctot!=savcrc) ERROR("SuperFile does not match sub files"); unsigned tr = (unsigned)(sfile->getFileSize(false,false)/srs); if (totrows!=tr) ERROR1("Superfile size does not match part sum %d",tr); sfile->detach(); sfile.clear(); sfile.setown(queryDistributedFileDirectory().lookupSuperFile("daregress::superfile1")); if (sfile) ERROR("Superfile deletion failed"); t = 37; for (i=0;i<100;i++) { s.clear().append("daregress::test").append(t); if (i%1) { Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(s.str()); if (!dfile) ERROR1("Could not find %s",s.str()); dfile->detach(); } else queryDistributedFileDirectory().removeEntry(s.str()); t = (t+37)%100; } printf("DFile removal complete\n"); t = 39; for (i=0;i<100;i++) { if (queryDistributedFileDirectory().exists(s.str())) ERROR1("Found %s after deletion",s.str()); Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(s.str()); if (dfile) ERROR1("Found %s after deletion",s.str()); t = (t+39)%100; } printf("DFile removal check complete\n"); queryNamedGroupStore().remove("daregress_group"); if (queryNamedGroupStore().lookup("daregress_group")) ERROR("Named group not removed"); }
void prepareKey(IDistributedFile *index) { IDistributedFile *f = index; IDistributedSuperFile *super = f->querySuperFile(); unsigned nparts = f->numParts(); // includes tlks if any, but unused in array performPartLookup.ensure(nparts); bool checkTLKConsistency = (nullptr != super) && !localKey && (0 != (TIRsorted & indexBaseHelper->getFlags())); if (nofilter) { while (nparts--) performPartLookup.append(true); if (!checkTLKConsistency) return; } else { while (nparts--) performPartLookup.append(false); // parts to perform lookup set later } Owned<IDistributedFileIterator> iter; if (super) { iter.setown(super->getSubFileIterator(true)); verifyex(iter->first()); f = &iter->query(); } unsigned width = f->numParts(); if (!localKey) --width; assertex(width); unsigned tlkCrc = 0; bool first = true; unsigned superSubIndex=0; bool fileCrc = false, rowCrc = false; for (;;) { Owned<IDistributedFilePart> part = f->getPart(width); if (checkTLKConsistency) { unsigned _tlkCrc; if (part->getCrc(_tlkCrc)) fileCrc = true; else if (part->queryAttributes().hasProp("@crc")) // NB: key "@crc" is not a crc on the file, but data within. { _tlkCrc = part->queryAttributes().getPropInt("@crc"); rowCrc = true; } else if (part->queryAttributes().hasProp("@tlkCrc")) // backward compat. { _tlkCrc = part->queryAttributes().getPropInt("@tlkCrc"); rowCrc = true; } else { if (rowCrc || fileCrc) { checkTLKConsistency = false; Owned<IException> e = MakeActivityWarning(&container, 0, "Cannot validate that tlks in superfile %s match, some crc attributes are missing", super->queryLogicalName()); queryJobChannel().fireException(e); } } if (rowCrc && fileCrc) { checkTLKConsistency = false; Owned<IException> e = MakeActivityWarning(&container, 0, "Cannot validate that tlks in superfile %s match, due to mixed crc types.", super->queryLogicalName()); queryJobChannel().fireException(e); } if (checkTLKConsistency) { if (first) { tlkCrc = _tlkCrc; first = false; } else if (tlkCrc != _tlkCrc) throw MakeActivityException(this, 0, "Sorted output on super files comprising of non coparitioned sub keys is not supported (TLK's do not match)"); } } if (!nofilter) { Owned<IKeyIndex> keyIndex; unsigned copy; for (copy=0; copy<part->numCopies(); copy++) { RemoteFilename rfn; OwnedIFile ifile = createIFile(part->getFilename(rfn,copy)); if (ifile->exists()) { StringBuffer remotePath; rfn.getRemotePath(remotePath); unsigned crc = 0; part->getCrc(crc); keyIndex.setown(createKeyIndex(remotePath.str(), crc, false, false)); break; } } if (!keyIndex) throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", index->queryLogicalName()); unsigned fixedSize = indexBaseHelper->queryDiskRecordSize()->querySerializedDiskMeta()->getFixedSize(); // used only if fixed Owned <IKeyManager> tlk = createLocalKeyManager(keyIndex, fixedSize, NULL); indexBaseHelper->createSegmentMonitors(tlk); tlk->finishSegmentMonitors(); tlk->reset(); while (tlk->lookup(false)) { if (tlk->queryFpos()) performPartLookup.replace(true, (aindex_t)(super?super->numSubFiles(true)*(tlk->queryFpos()-1)+superSubIndex:tlk->queryFpos()-1)); } } if (!super||!iter->next()) break; superSubIndex++; f = &iter->query(); if (width != f->numParts()-1) throw MakeActivityException(this, 0, "Super key %s, with mixture of sub key width are not supported.", f->queryLogicalName()); } }
int main(int argc, char* argv[]) { InitModuleObjects(); EnableSEHtoExceptionMapping(); if (argc<2) { usage(argv[0]); return -1; } bool forceGroupUpdate = false; StringBuffer filename; StringBuffer inst; StringBuffer dcat; StringBuffer dcomp; StringBuffer dinst; StringBuffer dip; for (int i=1;i<argc;i++) { if (argv[i][0]=='-') { if ((stricmp(argv[i],"-i")==0)&&(i+1<argc)) { inst.append(argv[++i]); } else if (0==stricmp(argv[i],"-f")) { forceGroupUpdate = true; } else if ((stricmp(argv[i],"-d")==0)&&(i+3<argc)) { dcat.append(argv[++i]); dcomp.append(argv[++i]); dinst.append(argv[++i]); } else if ((stricmp(argv[i],"-ip")==0)&&(i+1<argc)) { dip.append(argv[++i]); } else { usage(argv[0]); return -1; } } else { if (filename.length()) { usage(argv[0]); return -1; } filename.append(argv[i]); } } Owned<IPropertyTree> env; try { env.setown(createPTreeFromXMLFile(argv[1])); if (!env.get()) { fprintf(stderr,"Could not load Environment from %s\n",argv[1]); return 1; } const char *s = env->queryName(); if (!s||(strcmp(s,"Environment")!=0)) { fprintf(stderr,"File %s is invalid\n",argv[1]); return 1; } } catch (IException *e) { StringBuffer err; e->errorMessage(err); fprintf(stderr,"Could not load Environment from %s, %s\n",argv[1],err.str()); return 1; } int ret = 0; try { if (dcat.length()) { IPropertyTree* dirs = env->queryPropTree("Software/Directories"); StringBuffer dirout; if (getConfigurationDirectory(dirs,dcat.str(),dcomp.str(),dinst.str(),dirout)&&dirout.length()) { if (dip.length()) { SocketEndpoint ep(dip.str()); RemoteFilename rfn; rfn.setPath(ep,dirout.str()); rfn.getRemotePath(dirout.clear()); } printf("%s",dirout.str()); } else { ret = 1; } } else { if (!updateDaliEnv(env, forceGroupUpdate, inst.str())) ret = 1; } } catch (IException *e) { pexception("updtdalienv",e); e->Release(); ret = 1; } releaseAtoms(); return ret; }