void printKeywordsToXml() { StringBuffer buffer; unsigned int nGroups = sizeof(keywordList)/sizeof(keywordList[0]); buffer.append("<xml>\n"); for (unsigned i = 0; i < nGroups; ++i) { buffer.append(" <cat group=\"").append(keywordList[i].group).append("\">\n"); unsigned int j = 0; while(keywordList[i].keywords[j]) { buffer.append(" <keyword name=\"").append(keywordList[i].keywords[j]).append("\"/>\n"); ++j; } buffer.append(" </cat>\n"); } buffer.append("</xml>\n"); Owned<IFile> treeFile = createIFile("ECLKeywords.xml"); assertex(treeFile); Owned<IFileIO> io = treeFile->open(IFOcreaterw); assertex(io); Owned<IIOStream> out = createIOStream(io); assertex(out); out->write(buffer.length(), buffer.str()); }
void appendFile(IFileIO *srcio,IFileIOStream *out,bool sub) { static byte buf[BUFFER_SIZE]; Owned<IFileIOStream> srcstrm = createIOStream(srcio); loop { size32_t rd = srcstrm->read(BUFFER_SIZE,buf); if (!rd) break; out->write(rd,buf); } }
bool checkSavedFileCRC(IFile * ifile, bool & timesDiffer, unsigned & storedCrc) { StringBuffer s(ifile->queryFilename()); s.append(".crc"); Owned<IFile> crcFile = createIFile(s.str()); size32_t crcSz = (size32_t)crcFile->size(); Owned<IFileIO> crcIO = crcFile->open(IFOread); bool performCrc = false; timesDiffer = false; if (crcIO) { Owned<IFileIOStream> crcStream = createIOStream(crcIO); if (sizeof(storedCrc) == crcSz) // backward compat. if = in size to just crc (no date stamps) { verifyex(crcSz == crcStream->read(crcSz, &storedCrc)); performCrc = true; } else { size32_t sz; verifyex(sizeof(sz) == crcStream->read(sizeof(sz), &sz)); void *mem = malloc(sz); MemoryBuffer mb; mb.setBuffer(sz, mem, true); verifyex(sz == crcStream->read(sz, mem)); CDateTime storedCreateTime(mb); CDateTime storedModifiedTime(mb); CDateTime createTime, modifiedTime, accessedTime; ifile->getTime(&createTime, &modifiedTime, &accessedTime); if (!storedCreateTime.equals(createTime) || !storedModifiedTime.equals(modifiedTime)) timesDiffer = true; else { mb.read(storedCrc); performCrc = true; } } } return performCrc; }
void init (char const * filename, bool overwrite, size32_t _keyedsize, size32_t _rowsize, bool variableWidth, bool quickCompressed, unsigned nodeSize) { keyedsize = _keyedsize; rowsize = _rowsize; reccount = 0; keyFile.setown(createIFile(filename)); if(!overwrite && (keyFile->isFile() != notFound)) throw MakeStringException(0, "Found preexisting index file %s (overwrite not selected)", filename); keyFileIO.setown(keyFile->openShared(IFOcreate, IFSHfull)); // not sure if needs shared here if(!keyFileIO) throw MakeStringException(0, "Could not write index file %s", filename); keyStream.setown(createIOStream(keyFileIO)); unsigned flags = COL_PREFIX | HTREE_FULLSORT_KEY | HTREE_COMPRESSED_KEY; if(variableWidth) flags |= HTREE_VARSIZE; if(quickCompressed) flags |= HTREE_QUICK_COMPRESSED_KEY; keyBuilder.setown(createKeyBuilder(keyStream, flags, rowsize, nodeSize, keyedsize, 0)); // MORE - support for sequence other than 0... }
void CDiskWriteSlaveActivityBase::open() { if (dlfn.isExternal() && !firstNode()) { input.setown(createDataLinkSmartBuffer(this, inputs.item(0), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, false, &container.queryJob().queryIDiskUsage())); startInput(input); if (!rfsQueryParallel) { ActPrintLog("Blocked, waiting for previous part to complete write"); CMessageBuffer msg; if (!receiveMsg(msg, container.queryJob().queryMyRank()-1, mpTag)) return; rowcount_t prevRows; msg.read(prevRows); msg.read(tempExternalName); // reuse temp filename, last node will rename ActPrintLog("Previous write row count = %"RCPF"d", prevRows); } } else { input.set(inputs.item(0)); startInput(input); } processed = THORDATALINK_STARTED; bool extend = 0 != (diskHelperBase->getFlags() & TDWextend); if (extend) ActPrintLog("Extending file %s", fName.get()); size32_t exclsz = 0; calcFileCrc = true; bool external = dlfn.isExternal(); bool query = dlfn.isQuery(); if (query && compress) UNIMPLEMENTED; bool direct = query || (external && !firstNode()); bool rename = !external || (!query && lastNode()); Owned<IFileIO> iFileIO = createMultipleWrite(this, *partDesc, exclsz, compress, extend||(external&&!query), ecomp, this, direct, rename, &abortSoon, (external&&!query) ? &tempExternalName : NULL); if (compress) { ActPrintLog("Performing row compression on output file: %s", fName.get()); calcFileCrc = false; } Owned<IFileIOStream> stream; if (wantRaw()) { outraw.setown(createBufferedIOStream(iFileIO)); stream.set(outraw); } else { stream.setown(createIOStream(iFileIO)); out.setown(createRowWriter(stream,::queryRowSerializer(input),::queryRowAllocator(input),grouped,calcFileCrc,false)); // flushed by close } CDfsLogicalFileName dlfn; dlfn.set(logicalFilename); if (extend || (external && !query)) stream->seek(0,IFSend); ActPrintLog("Created output stream for %s", fName.get()); }
void CDiskWriteSlaveActivityBase::open() { if (dlfn.isExternal() && !firstNode()) { input.setown(createDataLinkSmartBuffer(this, inputs.item(0), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, false, &container.queryJob().queryIDiskUsage())); startInput(input); if (!rfsQueryParallel) { ActPrintLog("Blocked, waiting for previous part to complete write"); CMessageBuffer msg; if (!receiveMsg(msg, queryJobChannel().queryMyRank()-1, mpTag)) return; rowcount_t prevRows; msg.read(prevRows); msg.read(tempExternalName); // reuse temp filename, last node will rename ActPrintLog("Previous write row count = %" RCPF "d", prevRows); } } else { input.set(inputs.item(0)); startInput(input); } processed = THORDATALINK_STARTED; bool extend = 0 != (diskHelperBase->getFlags() & TDWextend); if (extend) ActPrintLog("Extending file %s", fName.get()); /* Fixed length record size is used when outputting compressed stream to determine run-length compression vs default LZW compression. * NB: only for FLAT files, not CSV or XML */ size32_t diskRowMinSz = 0; IOutputMetaData *diskRowMeta = diskHelperBase->queryDiskRecordSize()->querySerializedDiskMeta(); if (diskRowMeta->isFixedSize() && (TAKdiskwrite == container.getKind())) { diskRowMinSz = diskRowMeta->getMinRecordSize(); if (grouped) diskRowMinSz += 1; } calcFileCrc = true; bool external = dlfn.isExternal(); bool query = dlfn.isQuery(); if (query && compress) UNIMPLEMENTED; unsigned twFlags = external ? TW_External : 0; if (query || (external && !firstNode())) twFlags |= TW_Direct; if (!external || (!query && lastNode())) twFlags |= TW_RenameToPrimary; if (extend||(external&&!query)) twFlags |= TW_Extend; Owned<IFileIO> iFileIO = createMultipleWrite(this, *partDesc, diskRowMinSz, twFlags, compress, ecomp, this, &abortSoon, (external&&!query) ? &tempExternalName : NULL); if (compress) { ActPrintLog("Performing row compression on output file: %s", fName.get()); // NB: block compressed output has implicit crc of 0, no need to calculate in row writer. calcFileCrc = false; } Owned<IFileIOStream> stream; if (wantRaw()) { outraw.setown(createBufferedIOStream(iFileIO)); stream.set(outraw); } else { stream.setown(createIOStream(iFileIO)); unsigned rwFlags = 0; if (grouped) rwFlags |= rw_grouped; if (calcFileCrc) rwFlags |= rw_crc; out.setown(createRowWriter(stream, ::queryRowInterfaces(input), rwFlags)); } if (extend || (external && !query)) stream->seek(0,IFSend); ActPrintLog("Created output stream for %s", fName.get()); }
void CFileSpraySoapBindingEx::downloadFile(IEspContext &context, CHttpRequest* request, CHttpResponse* response) { try { StringBuffer netAddressStr, osStr, pathStr, nameStr; request->getParameter("NetAddress", netAddressStr); request->getParameter("OS", osStr); request->getParameter("Path", pathStr); request->getParameter("Name", nameStr); #if 0 StringArray files; IProperties* params = request->queryParameters(); Owned<IPropertyIterator> iter = params->getIterator(); if (iter && iter->first()) { while (iter->isValid()) { const char *keyname=iter->getPropKey(); if (!keyname || strncmp(keyname, "Names", 5)) continue; files.append(params->queryProp(iter->getPropKey())); iter->next(); } } #endif if (netAddressStr.length() < 1) throw MakeStringException(ECLWATCH_INVALID_INPUT, "Network address not specified."); if (pathStr.length() < 1) throw MakeStringException(ECLWATCH_INVALID_INPUT, "Path not specified."); if (nameStr.length() < 1) throw MakeStringException(ECLWATCH_INVALID_INPUT,"File name not specified."); char pathSep = '/'; if ((osStr.length() > 1) && (atoi(osStr.str())== OS_WINDOWS)) { pathSep = '\\'; } pathStr.replace(pathSep=='\\'?'/':'\\', pathSep); if (*(pathStr.str() + pathStr.length() -1) != pathSep) pathStr.append( pathSep ); StringBuffer fullName; fullName.appendf("%s%s", pathStr.str(), nameStr.str()); StringBuffer headerStr("attachment;"); headerStr.appendf("filename=%s", nameStr.str()); RemoteFilename rfn; rfn.setRemotePath(fullName.str()); SocketEndpoint ep(netAddressStr.str()); rfn.setIp(ep); Owned<IFile> rFile = createIFile(rfn); if (!rFile) throw MakeStringException(ECLWATCH_CANNOT_OPEN_FILE,"Cannot open file %s.",fullName.str()); OwnedIFileIO rIO = rFile->openShared(IFOread,IFSHfull); if (!rIO) throw MakeStringException(ECLWATCH_CANNOT_READ_FILE,"Cannot read file %s.",fullName.str()); IFileIOStream* ioS = createIOStream(rIO); context.addCustomerHeader("Content-disposition", headerStr.str()); response->setContent(ioS); response->setContentType(HTTP_TYPE_OCTET_STREAM); response->send(); } catch(IException* e) { FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR); } return; }