void associateRemoteInputs(CIArrayOf<OrderedInput> & orderedInputs, unsigned numPriorityInputs) { //If we know for sure the primary input, then tag it as worth reading ahead - otherwise it will be dynamically set later. if (numPriorityInputs > 0) { OrderedInput & input0 = orderedInputs.item(0); //Only read ahead etc. if this is a real index - not if it is a join. if (!input0.isJoin()) { input0.setReadAhead(true); input0.setAlwaysReadExact(); } } //Work out the last input of known priority which is read remotely. unsigned maxPriorityRemote = numPriorityInputs; while ((maxPriorityRemote >= 2) && !orderedInputs.item(maxPriorityRemote-1).readsRowsRemotely()) maxPriorityRemote--; //If the second ordered input is known to be read remotely, then we want to send multiple seek requests at the same time. //MORE: Maybe we should consider doing this to all other inputs if only one priority input is known. if (maxPriorityRemote >= 2) { for (unsigned i=1; i < maxPriorityRemote; i++) { IMultipleStepSeekInfo * seekInfo = orderedInputs.item(i-1).createMutipleReadWrapper(); orderedInputs.item(i).createMultipleSeekWrapper(seekInfo); } } }
void removeAbortHandler(IAbortHandler & handler) { CriticalBlock c(abortCrit); ForEachItemInRev(idx, handlers) { if (handlers.item(idx).ihandler == &handler) { handlers.remove(idx); break; } } queryUninstallAbortHandler(); }
bool notifyOnAbort() { // DBGLOG("notifyOnAbort %x", GetCurrentThreadId()); // CriticalBlock c(abortCrit); You would think that this was needed, but it locks up. // If it needs to be threadsafe, have to clone the list or something bool doExit = false; ForEachItemInRev(idx, handlers) { if (handlers.item(idx).handle()) doExit = true; } // DBGLOG("notifyOnAbort returning %d", (int) doExit); return doExit; }
void RowTransformer::generateCopies(unsigned & seq, CIArrayOf<RowRecord> const & records) { sequence = seq++; ForEachItemIn(fieldNum, records) { RowRecord const & record = records.item(fieldNum); if(record.isToFpos()) { assertex(sequence == 0); if(record.isFromFpos()) keepFpos = true; else generateCopyToFpos(record); } else { if(record.isFromFpos()) generateCopyFromFpos(record); else generateSimpleCopy(seq, record); } }
void verifyFile(const char *name,CDateTime *cutoff) { Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(name,UNKNOWN_USER); if (!file) return; IPropertyTree &fileprops = file->queryAttributes(); bool blocked = false; if (file->isCompressed(&blocked)&&!blocked) return; if (stopped) return; StringBuffer dtstr; if (fileprops.getProp("@verified",dtstr)) { if (!cutoff) return; CDateTime dt; dt.setString(dtstr.str()); if (dt.compare(*cutoff)<=0) return; } list.kill(); unsigned width = file->numParts(); unsigned short port = getDaliServixPort(); try { Owned<IDistributedFilePart> testpart = file->getPart(0); SocketEndpoint ep(testpart->queryNode()->endpoint()); if (!dafilesrvips.verifyDaliFileServer(ep)) { StringBuffer ips; ep.getIpText(ips); PROGLOG("VERIFY: file %s, cannot run DAFILESRV on %s",name,ips.str()); return; } } catch (IException *e) { StringBuffer s; s.appendf("VERIFY: file %s",name); EXCLOG(e, s.str()); e->Release(); return; } for (unsigned idx=0;idx<width;idx++) { Owned<IDistributedFilePart> part = file->getPart(idx); for (unsigned copy = 0; copy < part->numCopies(); copy++) { if (stopped) return; unsigned reqcrc; if (!part->getCrc(reqcrc)) continue; SocketEndpoint ep(part->queryNode()->endpoint()); if (!dafilesrvips.verifyDaliFileServer(ep)) { StringBuffer ips; ep.getIpText(ips); PROGLOG("VERIFY: file %s, cannot run DAFILESRV on %s",name,ips.str()); continue; } RemoteFilename rfn; part->getFilename(rfn,copy); rfn.setPort(port); add(rfn,idx,copy,reqcrc); } } if (list.ordinality()==0) return; PROGLOG("VERIFY: file %s started",name); file.clear(); CriticalSection crit; class casyncfor: public CAsyncFor { CFileCrcList *parent; CriticalSection &crit; public: bool ok; casyncfor(CFileCrcList *_parent, CriticalSection &_crit) : crit(_crit) { parent = _parent; ok = true; } void Do(unsigned i) { CriticalBlock block(crit); if (parent->stopped) return; CFileCrcItem &item = parent->list.item(i); RemoteFilename &rfn = item.filename; Owned<IFile> partfile; StringBuffer eps; try { partfile.setown(createIFile(rfn)); // PROGLOG("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str()); if (partfile) { if (parent->stopped) return; CriticalUnblock unblock(crit); item.crc = partfile->getCRC(); } else ok = false; } catch (IException *e) { StringBuffer s; s.appendf("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str()); EXCLOG(e, s.str()); e->Release(); ok = false; } } } afor(this,crit); afor.For(list.ordinality(),50,false,true); ForEachItemIn(j,list) { CFileCrcItem &item = list.item(j); if (item.crc!=item.requiredcrc) { if (stopped) return; StringBuffer rfs; PROGLOG("VERIFY: FAILED %s (%x,%x) file %s",name,item.crc,item.requiredcrc,item.filename.getRemotePath(rfs).str()); afor.ok = false; } }