void CDiskPartHandlerBase::open()
{
unsigned location;
StringBuffer filePath;
if (!(globals->getPropBool("@autoCopyBackup", true)?ensurePrimary(&activity, *partDesc, iFile, location, filePath):getBestFilePart(&activity, *partDesc, iFile, location, filePath, &activity)))
{
StringBuffer locations;
IException *e = MakeActivityException(&activity, TE_FileNotFound, "No physical file part for logical file %s, found at given locations: %s (Error = %d)", activity.logicalFilename.get(), getFilePartLocations(*partDesc, locations).str(), GetLastError());
EXCLOG(e, NULL);
throw e;
}
filename.set(iFile->queryFilename());
ActPrintLog(&activity, "%s[part=%d]: reading physical file '%s' (logical file = %s)", kindStr, which, filePath.str(), activity.logicalFilename.get());
if (checkFileCrc)
{
CDateTime createTime, modifiedTime, accessedTime;
iFile->getTime(&createTime, &modifiedTime, &accessedTime);
const char *descModTimeStr = partDesc->queryProperties().queryProp("@modified");
CDateTime descModTime;
descModTime.setString(descModTimeStr);
if (!descModTime.equals(modifiedTime, false))
{
StringBuffer diskTimeStr;
ActPrintLog(&activity, "WARNING: file (%s); modified date stamps on disk (%s) are not equal to published modified data (%s)", filePath.str(), modifiedTime.getString(diskTimeStr).str(), descModTimeStr);
}
}
ActPrintLog(&activity, "%s[part=%d]: Base offset to %"I64F"d", kindStr, which, fileBaseOffset);
if (compressed)
{
ActPrintLog(&activity, "Reading %s compressed file: %s", (NULL != activity.eexp.get())?"encrypted":blockCompressed?"block":"row", filename.get());
if (checkFileCrc)
{
checkFileCrc = false;
if (activity.crcCheckCompressed) // applies to encrypted too, (optional, default off)
{
ActPrintLog(&activity, "Calculating crc for file: %s", filename.get());
unsigned calcCrc = iFile->getCRC();
// NB: for compressed files should always be ~0
ActPrintLog(&activity, "Calculated crc = %x, storedCrc = %x", calcCrc, storedCrc);
if (calcCrc != storedCrc)
{
IThorException *e = MakeActivityException(&activity, TE_FileCrc, "CRC Failure validating compressed file: %s", iFile->queryFilename());
e->setAudience(MSGAUD_operator);
throw e;
}
}
}
}
}
void verifyFile(const char *name,CDateTime *cutoff)
{
Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(name,UNKNOWN_USER);
if (!file)
return;
IPropertyTree &fileprops = file->queryAttributes();
bool blocked = false;
if (file->isCompressed(&blocked)&&!blocked)
return;
if (stopped)
return;
StringBuffer dtstr;
if (fileprops.getProp("@verified",dtstr)) {
if (!cutoff)
return;
CDateTime dt;
dt.setString(dtstr.str());
if (dt.compare(*cutoff)<=0)
return;
}
list.kill();
unsigned width = file->numParts();
unsigned short port = getDaliServixPort();
try {
Owned<IDistributedFilePart> testpart = file->getPart(0);
SocketEndpoint ep(testpart->queryNode()->endpoint());
if (!dafilesrvips.verifyDaliFileServer(ep)) {
StringBuffer ips;
ep.getIpText(ips);
PROGLOG("VERIFY: file %s, cannot run DAFILESRV on %s",name,ips.str());
return;
}
}
catch (IException *e)
{
StringBuffer s;
s.appendf("VERIFY: file %s",name);
EXCLOG(e, s.str());
e->Release();
return;
}
for (unsigned idx=0;idx<width;idx++) {
Owned<IDistributedFilePart> part = file->getPart(idx);
for (unsigned copy = 0; copy < part->numCopies(); copy++) {
if (stopped)
return;
unsigned reqcrc;
if (!part->getCrc(reqcrc))
continue;
SocketEndpoint ep(part->queryNode()->endpoint());
if (!dafilesrvips.verifyDaliFileServer(ep)) {
StringBuffer ips;
ep.getIpText(ips);
PROGLOG("VERIFY: file %s, cannot run DAFILESRV on %s",name,ips.str());
continue;
}
RemoteFilename rfn;
part->getFilename(rfn,copy);
rfn.setPort(port);
add(rfn,idx,copy,reqcrc);
}
}
if (list.ordinality()==0)
return;
PROGLOG("VERIFY: file %s started",name);
file.clear();
CriticalSection crit;
class casyncfor: public CAsyncFor
{
CFileCrcList *parent;
CriticalSection &crit;
public:
bool ok;
casyncfor(CFileCrcList *_parent, CriticalSection &_crit)
: crit(_crit)
{
parent = _parent;
ok = true;
}
void Do(unsigned i)
{
CriticalBlock block(crit);
if (parent->stopped)
return;
CFileCrcItem &item = parent->list.item(i);
RemoteFilename &rfn = item.filename;
Owned<IFile> partfile;
StringBuffer eps;
try
{
partfile.setown(createIFile(rfn));
// PROGLOG("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str());
if (partfile) {
if (parent->stopped)
return;
CriticalUnblock unblock(crit);
item.crc = partfile->getCRC();
}
else
ok = false;
}
catch (IException *e)
{
StringBuffer s;
s.appendf("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str());
EXCLOG(e, s.str());
e->Release();
ok = false;
}
}
} afor(this,crit);
afor.For(list.ordinality(),50,false,true);
ForEachItemIn(j,list) {
CFileCrcItem &item = list.item(j);
if (item.crc!=item.requiredcrc) {
if (stopped)
return;
StringBuffer rfs;
PROGLOG("VERIFY: FAILED %s (%x,%x) file %s",name,item.crc,item.requiredcrc,item.filename.getRemotePath(rfs).str());
afor.ok = false;
}
}
