void CDiskPartHandlerBase::open() { unsigned location; StringBuffer filePath; if (!(globals->getPropBool("@autoCopyBackup", true)?ensurePrimary(&activity, *partDesc, iFile, location, filePath):getBestFilePart(&activity, *partDesc, iFile, location, filePath, &activity))) { StringBuffer locations; IException *e = MakeActivityException(&activity, TE_FileNotFound, "No physical file part for logical file %s, found at given locations: %s (Error = %d)", activity.logicalFilename.get(), getFilePartLocations(*partDesc, locations).str(), GetLastError()); EXCLOG(e, NULL); throw e; } filename.set(iFile->queryFilename()); ActPrintLog(&activity, "%s[part=%d]: reading physical file '%s' (logical file = %s)", kindStr, which, filePath.str(), activity.logicalFilename.get()); if (checkFileCrc) { CDateTime createTime, modifiedTime, accessedTime; iFile->getTime(&createTime, &modifiedTime, &accessedTime); const char *descModTimeStr = partDesc->queryProperties().queryProp("@modified"); CDateTime descModTime; descModTime.setString(descModTimeStr); if (!descModTime.equals(modifiedTime, false)) { StringBuffer diskTimeStr; ActPrintLog(&activity, "WARNING: file (%s); modified date stamps on disk (%s) are not equal to published modified data (%s)", filePath.str(), modifiedTime.getString(diskTimeStr).str(), descModTimeStr); } } ActPrintLog(&activity, "%s[part=%d]: Base offset to %"I64F"d", kindStr, which, fileBaseOffset); if (compressed) { ActPrintLog(&activity, "Reading %s compressed file: %s", (NULL != activity.eexp.get())?"encrypted":blockCompressed?"block":"row", filename.get()); if (checkFileCrc) { checkFileCrc = false; if (activity.crcCheckCompressed) // applies to encrypted too, (optional, default off) { ActPrintLog(&activity, "Calculating crc for file: %s", filename.get()); unsigned calcCrc = iFile->getCRC(); // NB: for compressed files should always be ~0 ActPrintLog(&activity, "Calculated crc = %x, storedCrc = %x", calcCrc, storedCrc); if (calcCrc != storedCrc) { IThorException *e = MakeActivityException(&activity, TE_FileCrc, "CRC Failure validating compressed file: %s", iFile->queryFilename()); e->setAudience(MSGAUD_operator); throw e; } } } } }
void verifyFile(const char *name,CDateTime *cutoff) { Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(name,UNKNOWN_USER); if (!file) return; IPropertyTree &fileprops = file->queryAttributes(); bool blocked = false; if (file->isCompressed(&blocked)&&!blocked) return; if (stopped) return; StringBuffer dtstr; if (fileprops.getProp("@verified",dtstr)) { if (!cutoff) return; CDateTime dt; dt.setString(dtstr.str()); if (dt.compare(*cutoff)<=0) return; } list.kill(); unsigned width = file->numParts(); unsigned short port = getDaliServixPort(); try { Owned<IDistributedFilePart> testpart = file->getPart(0); SocketEndpoint ep(testpart->queryNode()->endpoint()); if (!dafilesrvips.verifyDaliFileServer(ep)) { StringBuffer ips; ep.getIpText(ips); PROGLOG("VERIFY: file %s, cannot run DAFILESRV on %s",name,ips.str()); return; } } catch (IException *e) { StringBuffer s; s.appendf("VERIFY: file %s",name); EXCLOG(e, s.str()); e->Release(); return; } for (unsigned idx=0;idx<width;idx++) { Owned<IDistributedFilePart> part = file->getPart(idx); for (unsigned copy = 0; copy < part->numCopies(); copy++) { if (stopped) return; unsigned reqcrc; if (!part->getCrc(reqcrc)) continue; SocketEndpoint ep(part->queryNode()->endpoint()); if (!dafilesrvips.verifyDaliFileServer(ep)) { StringBuffer ips; ep.getIpText(ips); PROGLOG("VERIFY: file %s, cannot run DAFILESRV on %s",name,ips.str()); continue; } RemoteFilename rfn; part->getFilename(rfn,copy); rfn.setPort(port); add(rfn,idx,copy,reqcrc); } } if (list.ordinality()==0) return; PROGLOG("VERIFY: file %s started",name); file.clear(); CriticalSection crit; class casyncfor: public CAsyncFor { CFileCrcList *parent; CriticalSection &crit; public: bool ok; casyncfor(CFileCrcList *_parent, CriticalSection &_crit) : crit(_crit) { parent = _parent; ok = true; } void Do(unsigned i) { CriticalBlock block(crit); if (parent->stopped) return; CFileCrcItem &item = parent->list.item(i); RemoteFilename &rfn = item.filename; Owned<IFile> partfile; StringBuffer eps; try { partfile.setown(createIFile(rfn)); // PROGLOG("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str()); if (partfile) { if (parent->stopped) return; CriticalUnblock unblock(crit); item.crc = partfile->getCRC(); } else ok = false; } catch (IException *e) { StringBuffer s; s.appendf("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str()); EXCLOG(e, s.str()); e->Release(); ok = false; } } } afor(this,crit); afor.For(list.ordinality(),50,false,true); ForEachItemIn(j,list) { CFileCrcItem &item = list.item(j); if (item.crc!=item.requiredcrc) { if (stopped) return; StringBuffer rfs; PROGLOG("VERIFY: FAILED %s (%x,%x) file %s",name,item.crc,item.requiredcrc,item.filename.getRemotePath(rfs).str()); afor.ok = false; } }