virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override { isLocal = 0 != (TIWlocal & helper->getFlags()); mpTag = container.queryJobChannel().deserializeMPTag(data); mpTag2 = container.queryJobChannel().deserializeMPTag(data); data.read(active); if (active) { data.read(logicalFilename); partDesc.setown(deserializePartFileDescriptor(data)); } data.read(singlePartKey); data.read(refactor); if (singlePartKey) buildTlk = false; else { data.read(buildTlk); if (firstNode()) { if (buildTlk) tlkDesc.setown(deserializePartFileDescriptor(data)); else if (!isLocal) // existing tlk then.. { tlkDesc.setown(deserializePartFileDescriptor(data)); unsigned c; data.read(c); while (c--) { RemoteFilename rf; rf.deserialize(data); if (!existingTlkIFile) { Owned<IFile> iFile = createIFile(rf); if (iFile->exists()) existingTlkIFile.set(iFile); } } if (!existingTlkIFile) throw MakeActivityException(this, TE_FileNotFound, "Top level key part does not exist, for key"); } } } IOutputMetaData * diskSize = helper->queryDiskRecordSize(); assertex(!(diskSize->getMetaFlags() & MDFneedserializedisk)); if (diskSize->isVariableSize()) { if (TIWmaxlength & helper->getFlags()) maxDiskRecordSize = helper->getMaxKeySize(); else maxDiskRecordSize = KEYBUILD_MAXLENGTH; //Current default behaviour, could be improved in the future } else maxDiskRecordSize = diskSize->getFixedSize(); reportOverflow = false; }
virtual void init() { CMasterActivity::init(); OwnedRoxieString helperFileName = helper->getFileName(); StringBuffer expandedFileName; queryThorFileManager().addScope(container.queryJob(), helperFileName, expandedFileName, false); fileName.set(expandedFileName); dlfn.set(fileName); isLocal = 0 != (TIWlocal & helper->getFlags()); IOutputMetaData * diskSize = helper->queryDiskRecordSize(); unsigned minSize = diskSize->getMinRecordSize(); if (minSize > KEYBUILD_MAXLENGTH) throw MakeActivityException(this, 0, "Index minimum record length (%d) exceeds %d internal limit", minSize, KEYBUILD_MAXLENGTH); unsigned maxSize; if (diskSize->isVariableSize()) { if (TIWmaxlength & helper->getFlags()) maxSize = helper->getMaxKeySize(); else maxSize = KEYBUILD_MAXLENGTH; //Current default behaviour, could be improved in the future } else maxSize = diskSize->getFixedSize(); if (maxSize > KEYBUILD_MAXLENGTH) throw MakeActivityException(this, 0, "Index maximum record length (%d) exceeds %d internal limit. Maximum size = %d, try adjusting index MAXLENGTH", maxSize, KEYBUILD_MAXLENGTH, minSize); singlePartKey = 0 != (helper->getFlags() & TIWsmall) || dlfn.isExternal(); clusters.kill(); unsigned idx=0; while (true) { OwnedRoxieString cluster(helper->getCluster(idx)); if(!cluster) break; clusters.append(cluster); idx++; } IArrayOf<IGroup> groups; if (singlePartKey) { isLocal = true; buildTlk = false; } else if (!isLocal || globals->getPropBool("@buildLocalTlks", true)) buildTlk = true; fillClusterArray(container.queryJob(), fileName, clusters, groups); unsigned restrictedWidth = 0; if (TIWhaswidth & helper->getFlags()) { restrictedWidth = helper->getWidth(); if (restrictedWidth > container.queryJob().querySlaves()) throw MakeActivityException(this, 0, "Unsupported, can't refactor to width(%d) larger than host cluster(%d)", restrictedWidth, container.queryJob().querySlaves()); else if (restrictedWidth < container.queryJob().querySlaves()) { if (!isLocal) throw MakeActivityException(this, 0, "Unsupported, refactoring to few parts only supported for local indexes."); assertex(!singlePartKey); unsigned gwidth = groups.item(0).ordinality(); if (0 != container.queryJob().querySlaves() % gwidth) throw MakeActivityException(this, 0, "Unsupported, refactored target size (%d) must be factor of thor cluster width (%d)", groups.item(0).ordinality(), container.queryJob().querySlaves()); if (0 == restrictedWidth) restrictedWidth = gwidth; ForEachItemIn(g, groups) { IGroup &group = groups.item(g); if (gwidth != groups.item(g).ordinality()) throw MakeActivityException(this, 0, "Unsupported, cannot output multiple refactored widths, targeting cluster '%s' and '%s'", clusters.item(0), clusters.item(g)); if (gwidth != restrictedWidth) groups.replace(*group.subset((unsigned)0, restrictedWidth), g); } refactor = true; }