static void emptyScopes()
{
PROGLOG("Removing empty scopes");
Owned<IDFScopeIterator> iter = queryDistributedFileDirectory().getScopeIterator(NULL,true,true);
CDfsLogicalFileName dlfn;
StringBuffer s;
StringArray toremove;
ForEach(*iter) {
CDfsLogicalFileName dlfn;
StringBuffer scope;
scope.append(iter->query());
dlfn.set(scope.str(),"x");
dlfn.makeScopeQuery(s.clear(),true);
Owned<IRemoteConnection> conn = querySDS().connect(s.str(),myProcessSession(),RTM_LOCK_READ, INFINITE);
if (!conn)
DBGLOG("Could not connect to '%s' using %s",iter->query(),s.str());
else {
if (recursiveCheckEmptyScope(*conn->queryRoot())) {
PROGLOG("Empty scope %s",iter->query());
toremove.append(iter->query());
}
}
}
iter.clear();
ForEachItemIn(i,toremove) {
PROGLOG("Removed scope %s",toremove.item(i));
queryDistributedFileDirectory().removeEmptyScope(toremove.item(i));
}
bool LogicFileWrapper::doCompressFile(const char* name,StringBuffer& returnStr, IUserDescriptor* udesc)
{
try
{
Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(name, udesc) ;
if(!df)
return false;
ErrorReceiver err;
TaskQueue tq(200,&err);
Owned<IDistributedFilePartIterator> pi = df->getIterator();
ForEach(*pi)
{
tq.put(new CompressTask(&pi->query()));
}
tq.join();
err.getErrors(returnStr);
}
catch(IException* e){
StringBuffer msg;
e->errorMessage(msg);
WARNLOG("%s", msg.str());
e->Release();
}
catch(...){
WARNLOG("Unknown Exception caught within doCompressFile");
}
return true;
}
bool isFileKnownOnCluster(const char *logicalname, IConstWUClusterInfo *clusterInfo, IUserDescriptor* userdesc)
{
Owned<IDistributedFile> dst = queryDistributedFileDirectory().lookup(logicalname, userdesc, true);
if (dst)
{
SCMStringBuffer processName;
clusterInfo->getRoxieProcess(processName);
if (dst->findCluster(processName.str()) != NotFound)
return true; // file already known for this cluster
}
return false;
}
IndexDataSource::IndexDataSource(const char * _logicalName, IHqlExpression * _diskRecord, const char* _username, const char* _password)
{
logicalName.set(_logicalName);
diskRecord.set(_diskRecord);
Owned<IUserDescriptor> udesc;
if(_username != NULL && *_username != '\0')
{
udesc.setown(createUserDescriptor());
udesc->set(_username, _password);
}
df.setown(queryDistributedFileDirectory().lookup(logicalName, udesc.get()));
filtered = false;
}
void LogicFileWrapper::FindClusterName(const char* logicalName, StringBuffer& returnCluster, IUserDescriptor* udesc)
{
try {
Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName, udesc) ;
if(!df)
throw MakeStringException(-1,"Could not find logical file");
df->getClusterName(0,returnCluster); // ** TBD other cluster
}
catch(IException* e){
StringBuffer msg;
e->errorMessage(msg);
WARNLOG("%s", msg.str());
e->Release();
}
catch(...){
WARNLOG("Unknown Exception thrown within LogicFileWrapper::FindClusterName");
}
}
int init(const char *caller,const char *_path, bool allscopes,bool allfiles)
{
if (_path==NULL)
path.set("");
else
path.set(_path);
err = 0;
try {
if (!queryDistributedFileDirectory().loadScopeContents(_path,allscopes?&scopes:NULL,allfiles?&supers:NULL,allfiles?&files:NULL,true))
err = -ENOENT;
else
if (allfiles)
donefiles = true;
}
catch (IException *e) {
EXCLOG(e,caller);
err = -EFAULT;
}
return err;
}
bool LogicFileWrapper::doDeleteFile(const char* logicalName,const char *cluster, StringBuffer& returnStr, IUserDescriptor* udesc)
{
CDfsLogicalFileName lfn;
lfn.set(logicalName);
StringBuffer cname;
lfn.getCluster(cname);
if (0 == cname.length()) // if has no cluster, use supplied cluster
lfn.setCluster(cluster);
lfn.get(cname.clear(), false, true); // get file@cluster form;
try
{
IDistributedFileDirectory &fdir = queryDistributedFileDirectory();
{
Owned<IDistributedFile> df = fdir.lookup(cname.str(), udesc, true) ;
if(!df)
{
returnStr.appendf("<Message><Value>File %s not found</Value></Message>", cname.str());
return false;
}
}
fdir.removeEntry(cname.str(), udesc, NULL, REMOVE_FILE_SDS_CONNECT_TIMEOUT, true);
returnStr.appendf("<Message><Value>Deleted File %s</Value></Message>", cname.str());
DBGLOG("%s", returnStr.str());
return true;
}
catch (IException *e)
{
StringBuffer errorMsg;
e->errorMessage(returnStr);
e->Release();
PROGLOG("%s", errorMsg.str());
returnStr.appendf("<Message><Value>Failed to delete File %s, error: %s</Value></Message>", cname.str(), errorMsg.str());
}
return false;
}
void testMultiCluster()
{
Owned<IGroup> grp1 = createIGroup("192.168.51.1-5");
Owned<IGroup> grp2 = createIGroup("192.168.16.1-5");
Owned<IGroup> grp3 = createIGroup("192.168.53.1-5");
queryNamedGroupStore().add("testgrp1",grp1);
queryNamedGroupStore().add("testgrp2",grp2);
queryNamedGroupStore().add("testgrp3",grp3);
Owned<IFileDescriptor> fdesc = createFileDescriptor();
fdesc->setDefaultDir("/c$/thordata/test");
fdesc->setPartMask("testfile1._$P$_of_$N$");
fdesc->setNumParts(5);
ClusterPartDiskMapSpec mapping;
fdesc->addCluster(grp1,mapping);
fdesc->addCluster(grp2,mapping);
fdesc->addCluster(grp3,mapping);
queryDistributedFileDirectory().removeEntry("test::testfile1",UNKNOWN_USER);
Owned<IDistributedFile> file = queryDistributedFileDirectory().createNew(fdesc);
queryDistributedFileDirectory().removeEntry("test::testfile1",UNKNOWN_USER);
file->attach("test::testfile1",UNKNOWN_USER);
StringBuffer name;
unsigned i;
for (i=0;i<file->numClusters();i++)
PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
file.clear();
file.setown(queryDistributedFileDirectory().lookup("test::testfile1",UNKNOWN_USER));
for (i=0;i<file->numClusters();i++)
PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
file.clear();
file.setown(queryDistributedFileDirectory().lookup("test::testfile1@testgrp1",UNKNOWN_USER));
for (i=0;i<file->numClusters();i++)
PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
file.clear();
queryDistributedFileDirectory().removePhysical("test::testfile1@testgrp2",UNKNOWN_USER);
file.setown(queryDistributedFileDirectory().lookup("test::testfile1",UNKNOWN_USER));
for (i=0;i<file->numClusters();i++)
PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
}
static void test2()
{
const size32_t recsize = 17;
printf("Test DFS\n");
StringBuffer s;
unsigned i;
unsigned n;
unsigned t;
queryNamedGroupStore().remove("daregress_group");
queryDistributedFileDirectory().removeEntry("daregress::superfile1");
SocketEndpointArray epa;
for (n=0;n<400;n++) {
s.clear().append("192.168.").append(n/256).append('.').append(n%256);
SocketEndpoint ep(s.str());
epa.append(ep);
}
Owned<IGroup> group = createIGroup(epa);
queryNamedGroupStore().add("daregress_group",group,true);
if (!queryNamedGroupStore().find(group,s.clear()))
ERROR("Created logical group not found");
if (stricmp(s.str(),"daregress_group")!=0)
ERROR("Created logical group found with wrong name");
group.setown(queryNamedGroupStore().lookup("daregress_group"));
if (!group)
ERROR("named group lookup failed");
printf("Named group created - 400 nodes\n");
for (i=0;i<100;i++) {
Owned<IPropertyTree> pp = createPTree("Part");
Owned<IFileDescriptor>fdesc = createFileDescriptor();
fdesc->setDefaultDir("c:\\thordata\\regress");
n = 9;
for (unsigned k=0;k<400;k++) {
s.clear().append("192.168.").append(n/256).append('.').append(n%256);
Owned<INode> node = createINode(s.str());
pp->setPropInt64("@size",(n*777+i)*recsize);
s.clear().append("daregress_test").append(i).append("._").append(n+1).append("_of_400");
fdesc->setPart(n,node,s.str(),pp);
n = (n+9)%400;
}
fdesc->queryProperties().setPropInt("@recordSize",17);
s.clear().append("daregress::test").append(i);
queryDistributedFileDirectory().removeEntry(s.str());
StringBuffer cname;
Owned<IDistributedFile> dfile = queryDistributedFileDirectory().createNew(fdesc);
if (stricmp(dfile->getClusterName(0,cname),"daregress_group")!=0)
ERROR1("Cluster name wrong %d",i);
s.clear().append("daregress::test").append(i);
dfile->attach(s.str());
}
printf("DFile create done - 100 files\n");
unsigned samples = 5;
t = 33;
for (i=0;i<100;i++) {
s.clear().append("daregress::test").append(t);
if (!queryDistributedFileDirectory().exists(s.str()))
ERROR1("Could not find %s",s.str());
Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(s.str());
if (!dfile) {
ERROR1("Could not find %s",s.str());
continue;
}
offset_t totsz = 0;
n = 11;
for (unsigned k=0;k<400;k++) {
Owned<IDistributedFilePart> part = dfile->getPart(n);
if (!part) {
ERROR2("part not found %d %d",t,n);
continue;
}
s.clear().append("192.168.").append(n/256).append('.').append(n%256);
Owned<INode> node = createINode(s.str());
if (!node->equals(part->queryNode()))
ERROR2("part node mismatch %d, %d",t,n);
if (part->getFileSize(false,false)!=(n*777+t)*recsize)
ERROR4("size node mismatch %d, %d, %d, %d",t,n,(unsigned)part->getFileSize(false,false),(n*777+t)*recsize);
s.clear().append("daregress_test").append(t).append("._").append(n+1).append("_of_400");
/* ** TBD
if (stricmp(s.str(),part->queryPartName())!=0)
ERROR4("part name mismatch %d, %d '%s' '%s'",t,n,s.str(),part->queryPartName());
*/
totsz += (n*777+t)*recsize;
if ((samples>0)&&(i+n+t==k)) {
samples--;
RemoteFilename rfn;
part->getFilename(rfn,samples%2);
StringBuffer fn;
rfn.getRemotePath(fn);
printf("SAMPLE: %d,%d %s\n",t,n,fn.str());
}
n = (n+11)%400;
}
if (totsz!=dfile->getFileSize(false,false))
ERROR1("total size mismatch %d",t);
t = (t+33)%100;
}
printf("DFile lookup done - 100 files\n");
// check iteration
__int64 crctot = 0;
unsigned np = 0;
unsigned totrows = 0;
Owned<IDistributedFileIterator> fiter = queryDistributedFileDirectory().getIterator("daregress::*",false);
Owned<IDistributedFilePartIterator> piter;
ForEach(*fiter) {
piter.setown(fiter->query().getIterator());
ForEach(*piter) {
RemoteFilename rfn;
StringBuffer s;
piter->query().getFilename(rfn,0);
rfn.getRemotePath(s);
piter->query().getFilename(rfn,1);
rfn.getRemotePath(s);
crctot += crc32(s.str(),s.length(),0);
np++;
totrows += (unsigned)(piter->query().getFileSize(false,false)/fiter->query().queryProperties().getPropInt("@recordSize",-1));
}
}
piter.clear();
fiter.clear();
printf("DFile iterate done - %d parts, %d rows, CRC sum %"I64F"d\n",np,totrows,crctot);
Owned<IDistributedSuperFile> sfile;
sfile.setown(queryDistributedFileDirectory().createSuperFile("daregress::superfile1",true));
for (i = 0;i<100;i++) {
s.clear().append("daregress::test").append(i);
sfile->addSubFile(s.str());
}
sfile.clear();
sfile.setown(queryDistributedFileDirectory().lookupSuperFile("daregress::superfile1"));
if (!sfile) {
ERROR("Could not find added superfile");
return;
}
__int64 savcrc = crctot;
crctot = 0;
np = 0;
totrows = 0;
size32_t srs = (size32_t)sfile->queryProperties().getPropInt("@recordSize",-1);
if (srs!=17)
ERROR1("Superfile does not match subfile row size %d",srs);
piter.setown(sfile->getIterator());
ForEach(*piter) {
RemoteFilename rfn;
StringBuffer s;
piter->query().getFilename(rfn,0);
rfn.getRemotePath(s);
piter->query().getFilename(rfn,1);
rfn.getRemotePath(s);
crctot += crc32(s.str(),s.length(),0);
np++;
totrows += (unsigned)(piter->query().getFileSize(false,false)/srs);
}
piter.clear();
printf("Superfile iterate done - %d parts, %d rows, CRC sum %"I64F"d\n",np,totrows,crctot);
if (crctot!=savcrc)
ERROR("SuperFile does not match sub files");
unsigned tr = (unsigned)(sfile->getFileSize(false,false)/srs);
if (totrows!=tr)
ERROR1("Superfile size does not match part sum %d",tr);
sfile->detach();
sfile.clear();
sfile.setown(queryDistributedFileDirectory().lookupSuperFile("daregress::superfile1"));
if (sfile)
ERROR("Superfile deletion failed");
t = 37;
for (i=0;i<100;i++) {
s.clear().append("daregress::test").append(t);
if (i%1) {
Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(s.str());
if (!dfile)
ERROR1("Could not find %s",s.str());
dfile->detach();
}
else
queryDistributedFileDirectory().removeEntry(s.str());
t = (t+37)%100;
}
printf("DFile removal complete\n");
t = 39;
for (i=0;i<100;i++) {
if (queryDistributedFileDirectory().exists(s.str()))
ERROR1("Found %s after deletion",s.str());
Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(s.str());
if (dfile)
ERROR1("Found %s after deletion",s.str());
t = (t+39)%100;
}
printf("DFile removal check complete\n");
queryNamedGroupStore().remove("daregress_group");
if (queryNamedGroupStore().lookup("daregress_group"))
ERROR("Named group not removed");
}
void verifyFile(const char *name,CDateTime *cutoff)
{
Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(name,UNKNOWN_USER);
if (!file)
return;
IPropertyTree &fileprops = file->queryAttributes();
bool blocked = false;
if (file->isCompressed(&blocked)&&!blocked)
return;
if (stopped)
return;
StringBuffer dtstr;
if (fileprops.getProp("@verified",dtstr)) {
if (!cutoff)
return;
CDateTime dt;
dt.setString(dtstr.str());
if (dt.compare(*cutoff)<=0)
return;
}
list.kill();
unsigned width = file->numParts();
unsigned short port = getDaliServixPort();
try {
Owned<IDistributedFilePart> testpart = file->getPart(0);
SocketEndpoint ep(testpart->queryNode()->endpoint());
if (!dafilesrvips.verifyDaliFileServer(ep)) {
StringBuffer ips;
ep.getIpText(ips);
PROGLOG("VERIFY: file %s, cannot run DAFILESRV on %s",name,ips.str());
return;
}
}
catch (IException *e)
{
StringBuffer s;
s.appendf("VERIFY: file %s",name);
EXCLOG(e, s.str());
e->Release();
return;
}
for (unsigned idx=0;idx<width;idx++) {
Owned<IDistributedFilePart> part = file->getPart(idx);
for (unsigned copy = 0; copy < part->numCopies(); copy++) {
if (stopped)
return;
unsigned reqcrc;
if (!part->getCrc(reqcrc))
continue;
SocketEndpoint ep(part->queryNode()->endpoint());
if (!dafilesrvips.verifyDaliFileServer(ep)) {
StringBuffer ips;
ep.getIpText(ips);
PROGLOG("VERIFY: file %s, cannot run DAFILESRV on %s",name,ips.str());
continue;
}
RemoteFilename rfn;
part->getFilename(rfn,copy);
rfn.setPort(port);
add(rfn,idx,copy,reqcrc);
}
}
if (list.ordinality()==0)
return;
PROGLOG("VERIFY: file %s started",name);
file.clear();
CriticalSection crit;
class casyncfor: public CAsyncFor
{
CFileCrcList *parent;
CriticalSection &crit;
public:
bool ok;
casyncfor(CFileCrcList *_parent, CriticalSection &_crit)
: crit(_crit)
{
parent = _parent;
ok = true;
}
void Do(unsigned i)
{
CriticalBlock block(crit);
if (parent->stopped)
return;
CFileCrcItem &item = parent->list.item(i);
RemoteFilename &rfn = item.filename;
Owned<IFile> partfile;
StringBuffer eps;
try
{
partfile.setown(createIFile(rfn));
// PROGLOG("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str());
if (partfile) {
if (parent->stopped)
return;
CriticalUnblock unblock(crit);
item.crc = partfile->getCRC();
}
else
ok = false;
}
catch (IException *e)
{
StringBuffer s;
s.appendf("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str());
EXCLOG(e, s.str());
e->Release();
ok = false;
}
}
} afor(this,crit);
afor.For(list.ordinality(),50,false,true);
ForEachItemIn(j,list) {
CFileCrcItem &item = list.item(j);
if (item.crc!=item.requiredcrc) {
if (stopped)
return;
StringBuffer rfs;
PROGLOG("VERIFY: FAILED %s (%x,%x) file %s",name,item.crc,item.requiredcrc,item.filename.getRemotePath(rfs).str());
afor.ok = false;
}
}
int CFileSpraySoapBindingEx::onGetInstantQuery(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method)
{
bool permission = true;
bool bDownloadFile = false;
bool bProcess;
StringBuffer sourceLogicalFile;
StringBuffer methodbuf;
StringBuffer submethod;
StringBuffer xsltFileName(getCFD());
xsltFileName.append("smc_xslt/");
if (stricmp(method, "SprayFixedInput")==0)
{
if (!context.validateFeatureAccess(FILE_SPRAY_URL, SecAccess_Write, false))
permission = false;
bProcess = true;
xsltFileName.append("fs_sprayForm.xslt");
methodbuf.append("SprayFixed");
}
else if(stricmp(method, "SprayVariableInput")==0)
{
if (!context.validateFeatureAccess(FILE_SPRAY_URL, SecAccess_Write, false))
permission = false;
bProcess = true;
xsltFileName.append("fs_sprayForm.xslt");
methodbuf.append("SprayVariable");
request->getParameter("submethod", submethod);
}
else if (stricmp(method, "DesprayInput")==0)
{
if (!context.validateFeatureAccess(FILE_DESPRAY_URL, SecAccess_Write, false))
permission = false;
request->getParameter("sourceLogicalName", sourceLogicalFile);
xsltFileName.append("fs_desprayCopyForm.xslt");
methodbuf.append("Despray");
bProcess = true;
}
else if (stricmp(method, "CopyInput") == 0)
{
if (!context.validateFeatureAccess(FILE_SPRAY_URL, SecAccess_Write, false))
permission = false;
request->getParameter("sourceLogicalName", sourceLogicalFile);
xsltFileName.append("fs_desprayCopyForm.xslt");
methodbuf.append("Copy");
bProcess = true;
}
else if (stricmp(method, "RenameInput") == 0)
{
if (!context.validateFeatureAccess(FILE_SPRAY_URL, SecAccess_Write, false))
permission = false;
request->getParameter("sourceLogicalName", sourceLogicalFile);
xsltFileName.append("fs_renameForm.xslt");
methodbuf.append("Rename");
bProcess = true;
}
else if (stricmp(method, "DownloadFile") == 0)
{
if (!context.validateFeatureAccess(FILE_SPRAY_URL, SecAccess_Full, false))
permission = false;
downloadFile(context, request, response);
bDownloadFile = true;
bProcess = true;
}
else
bProcess = false;
if (bProcess)
{
if (bDownloadFile)
return 0;
StringBuffer xml;
Owned<IProperties> params(createProperties());
if (!permission)
{
params->setProp("@method", methodbuf.str());
xml.append("<Environment><ErrorMessage>Permission denied.</ErrorMessage></Environment>");
}
else
{
if(submethod.length() > 0)
params->setProp("@submethod", submethod.str());
params->setProp("@method", methodbuf.str());
if (*sourceLogicalFile.str())
{
params->setProp("@sourceLogicalName", sourceLogicalFile.str());
Owned<IUserDescriptor> userdesc;
StringBuffer username;
context.getUserID(username);
if(username.length() > 0)
{
const char* passwd = context.queryPassword();
userdesc.setown(createUserDescriptor());
userdesc->set(username.str(), passwd);
try
{
if (stricmp(method, "CopyInput") == 0)
{
Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(sourceLogicalFile.str(), userdesc.get());
if(!df)
{
throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"Could not find file %s.",sourceLogicalFile.str());
}
const char *kind = df->queryAttributes().queryProp("@kind");
if (kind && strcmp(kind,"key")==0)
{
params->setProp("@compressflag", 0);
}
else if(df->isCompressed())
{
params->setProp("@compressflag", 2);
}
else
{
params->setProp("@compressflag", 1);
}
}
}
catch (IException *E)
{
Owned<IXslProcessor> xslp = getXslProcessor();
if (!xslp)
throw E;
Owned<IMultiException> me = MakeMultiException();
me->append(*E);
response->handleExceptions(xslp, me, "FileSpray", method, StringBuffer(getCFD()).append("./smc_xslt/exceptions.xslt").str());
return 0;
}
}
}
else
{
params->setProp("@compressflag", 1);
}
StringBuffer wuid;
request->getParameter("wuid", wuid);
Owned<IPropertyTree> pTree = createPTreeForXslt(method, wuid.str());
toXML(pTree, xml, false);
}
IProperties* requestparams = request->queryParameters();
if(requestparams && requestparams->hasProp("rawxml_"))
{
response->setContent(xml.str());
response->setContentType(HTTP_TYPE_APPLICATION_XML);
}
else{
StringBuffer htmlbuf;
xsltTransform(xml.str(), xsltFileName.str(), params, htmlbuf);
response->setContent(htmlbuf.str());
response->setContentType(HTTP_TYPE_TEXT_HTML_UTF8);
}
response->send();
return 0;
}
else
return CFileSpraySoapBinding::onGetInstantQuery(context, request, response, service, method);
}
