void printKeywordsToXml()
{
StringBuffer buffer;
unsigned int nGroups = sizeof(keywordList)/sizeof(keywordList[0]);
buffer.append("<xml>\n");
for (unsigned i = 0; i < nGroups; ++i)
{
buffer.append(" <cat group=\"").append(keywordList[i].group).append("\">\n");
unsigned int j = 0;
while(keywordList[i].keywords[j])
{
buffer.append(" <keyword name=\"").append(keywordList[i].keywords[j]).append("\"/>\n");
++j;
}
buffer.append(" </cat>\n");
}
buffer.append("</xml>\n");
Owned<IFile> treeFile = createIFile("ECLKeywords.xml");
assertex(treeFile);
Owned<IFileIO> io = treeFile->open(IFOcreaterw);
assertex(io);
Owned<IIOStream> out = createIOStream(io);
assertex(out);
out->write(buffer.length(), buffer.str());
}
void appendFile(IFileIO *srcio,IFileIOStream *out,bool sub)
{
static byte buf[BUFFER_SIZE];
Owned<IFileIOStream> srcstrm = createIOStream(srcio);
loop {
size32_t rd = srcstrm->read(BUFFER_SIZE,buf);
if (!rd)
break;
out->write(rd,buf);
}
}
bool checkSavedFileCRC(IFile * ifile, bool & timesDiffer, unsigned & storedCrc)
{
StringBuffer s(ifile->queryFilename());
s.append(".crc");
Owned<IFile> crcFile = createIFile(s.str());
size32_t crcSz = (size32_t)crcFile->size();
Owned<IFileIO> crcIO = crcFile->open(IFOread);
bool performCrc = false;
timesDiffer = false;
if (crcIO)
{
Owned<IFileIOStream> crcStream = createIOStream(crcIO);
if (sizeof(storedCrc) == crcSz) // backward compat. if = in size to just crc (no date stamps)
{
verifyex(crcSz == crcStream->read(crcSz, &storedCrc));
performCrc = true;
}
else
{
size32_t sz;
verifyex(sizeof(sz) == crcStream->read(sizeof(sz), &sz));
void *mem = malloc(sz);
MemoryBuffer mb;
mb.setBuffer(sz, mem, true);
verifyex(sz == crcStream->read(sz, mem));
CDateTime storedCreateTime(mb);
CDateTime storedModifiedTime(mb);
CDateTime createTime, modifiedTime, accessedTime;
ifile->getTime(&createTime, &modifiedTime, &accessedTime);
if (!storedCreateTime.equals(createTime) || !storedModifiedTime.equals(modifiedTime))
timesDiffer = true;
else
{
mb.read(storedCrc);
performCrc = true;
}
}
}
return performCrc;
}
void init (char const * filename, bool overwrite, size32_t _keyedsize, size32_t _rowsize, bool variableWidth, bool quickCompressed, unsigned nodeSize)
{
keyedsize = _keyedsize;
rowsize = _rowsize;
reccount = 0;
keyFile.setown(createIFile(filename));
if(!overwrite && (keyFile->isFile() != notFound))
throw MakeStringException(0, "Found preexisting index file %s (overwrite not selected)", filename);
keyFileIO.setown(keyFile->openShared(IFOcreate, IFSHfull)); // not sure if needs shared here
if(!keyFileIO)
throw MakeStringException(0, "Could not write index file %s", filename);
keyStream.setown(createIOStream(keyFileIO));
unsigned flags = COL_PREFIX | HTREE_FULLSORT_KEY | HTREE_COMPRESSED_KEY;
if(variableWidth)
flags |= HTREE_VARSIZE;
if(quickCompressed)
flags |= HTREE_QUICK_COMPRESSED_KEY;
keyBuilder.setown(createKeyBuilder(keyStream, flags, rowsize, nodeSize, keyedsize, 0)); // MORE - support for sequence other than 0...
}
void CDiskWriteSlaveActivityBase::open()
{
if (dlfn.isExternal() && !firstNode())
{
input.setown(createDataLinkSmartBuffer(this, inputs.item(0), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, false, &container.queryJob().queryIDiskUsage()));
startInput(input);
if (!rfsQueryParallel)
{
ActPrintLog("Blocked, waiting for previous part to complete write");
CMessageBuffer msg;
if (!receiveMsg(msg, container.queryJob().queryMyRank()-1, mpTag))
return;
rowcount_t prevRows;
msg.read(prevRows);
msg.read(tempExternalName); // reuse temp filename, last node will rename
ActPrintLog("Previous write row count = %"RCPF"d", prevRows);
}
}
else
{
input.set(inputs.item(0));
startInput(input);
}
processed = THORDATALINK_STARTED;
bool extend = 0 != (diskHelperBase->getFlags() & TDWextend);
if (extend)
ActPrintLog("Extending file %s", fName.get());
size32_t exclsz = 0;
calcFileCrc = true;
bool external = dlfn.isExternal();
bool query = dlfn.isQuery();
if (query && compress)
UNIMPLEMENTED;
bool direct = query || (external && !firstNode());
bool rename = !external || (!query && lastNode());
Owned<IFileIO> iFileIO = createMultipleWrite(this, *partDesc, exclsz, compress, extend||(external&&!query), ecomp, this, direct, rename, &abortSoon, (external&&!query) ? &tempExternalName : NULL);
if (compress)
{
ActPrintLog("Performing row compression on output file: %s", fName.get());
calcFileCrc = false;
}
Owned<IFileIOStream> stream;
if (wantRaw())
{
outraw.setown(createBufferedIOStream(iFileIO));
stream.set(outraw);
}
else
{
stream.setown(createIOStream(iFileIO));
out.setown(createRowWriter(stream,::queryRowSerializer(input),::queryRowAllocator(input),grouped,calcFileCrc,false)); // flushed by close
}
CDfsLogicalFileName dlfn;
dlfn.set(logicalFilename);
if (extend || (external && !query))
stream->seek(0,IFSend);
ActPrintLog("Created output stream for %s", fName.get());
}
void CDiskWriteSlaveActivityBase::open()
{
if (dlfn.isExternal() && !firstNode())
{
input.setown(createDataLinkSmartBuffer(this, inputs.item(0), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, false, &container.queryJob().queryIDiskUsage()));
startInput(input);
if (!rfsQueryParallel)
{
ActPrintLog("Blocked, waiting for previous part to complete write");
CMessageBuffer msg;
if (!receiveMsg(msg, queryJobChannel().queryMyRank()-1, mpTag))
return;
rowcount_t prevRows;
msg.read(prevRows);
msg.read(tempExternalName); // reuse temp filename, last node will rename
ActPrintLog("Previous write row count = %" RCPF "d", prevRows);
}
}
else
{
input.set(inputs.item(0));
startInput(input);
}
processed = THORDATALINK_STARTED;
bool extend = 0 != (diskHelperBase->getFlags() & TDWextend);
if (extend)
ActPrintLog("Extending file %s", fName.get());
/* Fixed length record size is used when outputting compressed stream to determine run-length compression vs default LZW compression.
* NB: only for FLAT files, not CSV or XML
*/
size32_t diskRowMinSz = 0;
IOutputMetaData *diskRowMeta = diskHelperBase->queryDiskRecordSize()->querySerializedDiskMeta();
if (diskRowMeta->isFixedSize() && (TAKdiskwrite == container.getKind()))
{
diskRowMinSz = diskRowMeta->getMinRecordSize();
if (grouped)
diskRowMinSz += 1;
}
calcFileCrc = true;
bool external = dlfn.isExternal();
bool query = dlfn.isQuery();
if (query && compress)
UNIMPLEMENTED;
unsigned twFlags = external ? TW_External : 0;
if (query || (external && !firstNode()))
twFlags |= TW_Direct;
if (!external || (!query && lastNode()))
twFlags |= TW_RenameToPrimary;
if (extend||(external&&!query))
twFlags |= TW_Extend;
Owned<IFileIO> iFileIO = createMultipleWrite(this, *partDesc, diskRowMinSz, twFlags, compress, ecomp, this, &abortSoon, (external&&!query) ? &tempExternalName : NULL);
if (compress)
{
ActPrintLog("Performing row compression on output file: %s", fName.get());
// NB: block compressed output has implicit crc of 0, no need to calculate in row writer.
calcFileCrc = false;
}
Owned<IFileIOStream> stream;
if (wantRaw())
{
outraw.setown(createBufferedIOStream(iFileIO));
stream.set(outraw);
}
else
{
stream.setown(createIOStream(iFileIO));
unsigned rwFlags = 0;
if (grouped)
rwFlags |= rw_grouped;
if (calcFileCrc)
rwFlags |= rw_crc;
out.setown(createRowWriter(stream, ::queryRowInterfaces(input), rwFlags));
}
if (extend || (external && !query))
stream->seek(0,IFSend);
ActPrintLog("Created output stream for %s", fName.get());
}
void CFileSpraySoapBindingEx::downloadFile(IEspContext &context, CHttpRequest* request, CHttpResponse* response)
{
try
{
StringBuffer netAddressStr, osStr, pathStr, nameStr;
request->getParameter("NetAddress", netAddressStr);
request->getParameter("OS", osStr);
request->getParameter("Path", pathStr);
request->getParameter("Name", nameStr);
#if 0
StringArray files;
IProperties* params = request->queryParameters();
Owned<IPropertyIterator> iter = params->getIterator();
if (iter && iter->first())
{
while (iter->isValid())
{
const char *keyname=iter->getPropKey();
if (!keyname || strncmp(keyname, "Names", 5))
continue;
files.append(params->queryProp(iter->getPropKey()));
iter->next();
}
}
#endif
if (netAddressStr.length() < 1)
throw MakeStringException(ECLWATCH_INVALID_INPUT, "Network address not specified.");
if (pathStr.length() < 1)
throw MakeStringException(ECLWATCH_INVALID_INPUT, "Path not specified.");
if (nameStr.length() < 1)
throw MakeStringException(ECLWATCH_INVALID_INPUT,"File name not specified.");
char pathSep = '/';
if ((osStr.length() > 1) && (atoi(osStr.str())== OS_WINDOWS))
{
pathSep = '\\';
}
pathStr.replace(pathSep=='\\'?'/':'\\', pathSep);
if (*(pathStr.str() + pathStr.length() -1) != pathSep)
pathStr.append( pathSep );
StringBuffer fullName;
fullName.appendf("%s%s", pathStr.str(), nameStr.str());
StringBuffer headerStr("attachment;");
headerStr.appendf("filename=%s", nameStr.str());
RemoteFilename rfn;
rfn.setRemotePath(fullName.str());
SocketEndpoint ep(netAddressStr.str());
rfn.setIp(ep);
Owned<IFile> rFile = createIFile(rfn);
if (!rFile)
throw MakeStringException(ECLWATCH_CANNOT_OPEN_FILE,"Cannot open file %s.",fullName.str());
OwnedIFileIO rIO = rFile->openShared(IFOread,IFSHfull);
if (!rIO)
throw MakeStringException(ECLWATCH_CANNOT_READ_FILE,"Cannot read file %s.",fullName.str());
IFileIOStream* ioS = createIOStream(rIO);
context.addCustomerHeader("Content-disposition", headerStr.str());
response->setContent(ioS);
response->setContentType(HTTP_TYPE_OCTET_STREAM);
response->send();
}
catch(IException* e)
{
FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
}
return;
}
