void CDiskWriteSlaveActivityBase::init(MemoryBuffer &data, MemoryBuffer &slaveData) { diskHelperBase = static_cast <IHThorDiskWriteArg *> (queryHelper()); data.read(logicalFilename); dlfn.set(logicalFilename); if (diskHelperBase->getFlags() & TDXtemporary) data.read(usageCount); if (diskHelperBase->getFlags() & TDWextend) { assertex(!"TBD extended CRC broken"); unsigned crc; if (partDesc->getCrc(crc)) fileCRC.reset(~crc); } partDesc.setown(deserializePartFileDescriptor(data)); if (dlfn.isExternal()) { mpTag = container.queryJob().deserializeMPTag(data); if (dlfn.isQuery() && (0 != container.queryJob().getWorkUnitValueInt("rfsqueryparallel", 0))) rfsQueryParallel = true; } if (0 != (diskHelperBase->getFlags() & TDXgrouped)) grouped = true; compress = partDesc->queryOwner().isCompressed(); void *ekey; size32_t ekeylen; diskHelperBase->getEncryptKey(ekeylen,ekey); if (ekeylen!=0) { ecomp.setown(createAESCompressor256(ekeylen,ekey)); memset(ekey,0,ekeylen); free(ekey); compress = true; } }
void open() { char drive [_MAX_DRIVE]; char dir [_MAX_DIR]; char fname [_MAX_DIR]; char ext [_MAX_EXT]; _splitpath(fileName.str(), drive, dir, fname, ext); StringBuffer directory; directory.append(drive).append(dir); Owned<IFile> cd = createIFile(directory.str()); cd->createDirectory(); IHThorSpillArg *helper = (IHThorSpillArg *)queryHelper(); void *ekey; size32_t ekeylen; helper->getEncryptKey(ekeylen,ekey); Owned<ICompressor> ecomp; if (ekeylen!=0) { ecomp.setown(createAESCompressor256(ekeylen,ekey)); memset(ekey,0,ekeylen); free(ekey); compress = true; } Owned<IFile> file = createIFile(fileName.str()); Owned<IFileIO> iFileIO; bool fixedRecordSize = queryRowMetaData()->isFixedSize(); size32_t minrecsize = queryRowMetaData()->getMinRecordSize(); if (fixedRecordSize) ActPrintLog("SPILL: created fixed output %s recsize=%u", (0!=ekeylen)?"[encrypted]":compress?"[compressed]":"",minrecsize); else ActPrintLog("SPILL: created variable output %s, minrecsize=%u", (0!=ekeylen)?"[encrypted]":compress?"[compressed]":"",minrecsize); unsigned rwFlags = (DEFAULT_RWFLAGS & ~rw_autoflush); // flushed by close() if (compress) rwFlags |= rw_compress; else rwFlags |= rw_crc; // only if !compress if (grouped) rwFlags |= rw_grouped; out.setown(createRowWriter(file, this, rwFlags)); }
void CDiskWriteSlaveActivityBase::process() { compress = partDesc->queryOwner().isCompressed(); void *ekey; size32_t ekeylen; diskHelperBase->getEncryptKey(ekeylen,ekey); if (ekeylen!=0) { ecomp.setown(createAESCompressor256(ekeylen,ekey)); memset(ekey,0,ekeylen); free(ekey); compress = true; } calcFileCrc = false; uncompressedBytesWritten = 0; replicateDone = 0; StringBuffer tmpStr; fName.set(getPartFilename(*partDesc, 0, tmpStr).str()); if (diskHelperBase->getFlags() & TDXtemporary && !container.queryJob().queryUseCheckpoints()) container.queryTempHandler()->registerFile(fName, container.queryOwner().queryGraphId(), usageCount, true); try { ActPrintLog("handling fname : %s", fName.get()); try { open(); assertex(out||outraw); write(); } catch (IException *) { abortSoon = true; try { close(); } catch (IException *e) { EXCLOG(e, "close()"); e->Release(); } throw; } catch (CATCHALL) { abortSoon = true; try { close(); } catch (IException *e) { EXCLOG(e, "close()"); e->Release(); } throw; } close(); } catch (IException *) { calcFileCrc = false; throw; } catch(CATCHALL) { calcFileCrc = false; throw; } unsigned crc = compress?~0:fileCRC.get(); ActPrintLog("Wrote %" RCPF "d records%s", processed & THORDATALINK_COUNT_MASK, calcFileCrc?StringBuffer(", crc=0x").appendf("%X", crc).str() : ""); }