/*! \todo */ NameSet Parameter::getDependentParameterNameSet() const throw() { NameSet lNameSet; for(NameSet::const_iterator lItr = mDependentParameterNameSet.begin(); lItr != mDependentParameterNameSet.end(); lItr++) lNameSet.insert(lItr->c_str()); return lNameSet; }
// Called before any call to GenericFilter::Preflight(). void EnvironmentBase::EnterPreflightPhase( ParamList* inParamList, StateList* inStateList, StateVector* /*inStateVector*/ ) { bcierr__.SetAction( BCIStream::ConfigurationError ); phase_ = preflight; Accessor_<ParamList>::spGlobal = inParamList; Accessor_<StateList>::spGlobal = inStateList; Accessor_<StateVector>::spGlobal = NULL; BCIStream::Apply( *inParamList ); ParamsRangeChecked().clear(); if( inParamList ) { NameSet notAutoConfig; for( int i = 0; i < inParamList->Size(); ++i ) { const Param& p = inParamList->ByIndex( i ); if( !IsAutoConfigParam( p ) ) notAutoConfig.insert( p.Name() ); } RangeCheckParams( inParamList, notAutoConfig ); } ParamsAccessedDuringPreflight().clear(); StatesAccessedDuringPreflight().clear(); for( ExtensionsContainer::iterator i = Extensions().begin(); i != Extensions().end(); ++i ) ( *i )->CallPreflight(); }
QStringList QWinSettingsPrivate::children(const QString &uKey, ChildSpec spec) const { NameSet result; QString rKey = escapedKey(uKey); for (int i = 0; i < regList.size(); ++i) { HKEY parent_handle = regList.at(i).handle(); if (parent_handle == 0) continue; HKEY handle = openKey(parent_handle, KEY_READ, rKey); if (handle == 0) continue; if (spec == AllKeys) { NameSet keys; allKeys(handle, QLatin1String(""), &keys); mergeKeySets(&result, keys); } else { // ChildGroups or ChildKeys QStringList names = childKeysOrGroups(handle, spec); mergeKeySets(&result, names); } RegCloseKey(handle); if (!fallbacks) return result.keys(); } return result.keys(); }
void IndexForNativeFormat::read(ReadBuffer & istr, const NameSet & required_columns) { while (!istr.eof()) { blocks.emplace_back(); IndexOfBlockForNativeFormat & block = blocks.back(); readVarUInt(block.num_columns, istr); readVarUInt(block.num_rows, istr); if (block.num_columns < required_columns.size()) throw Exception("Index contain less than required columns", ErrorCodes::INCORRECT_INDEX); for (size_t i = 0; i < block.num_columns; ++i) { IndexOfOneColumnForNativeFormat column_index; readBinary(column_index.name, istr); readBinary(column_index.type, istr); readBinary(column_index.location.offset_in_compressed_file, istr); readBinary(column_index.location.offset_in_decompressed_block, istr); if (required_columns.count(column_index.name)) block.columns.push_back(std::move(column_index)); } if (block.columns.size() < required_columns.size()) throw Exception("Index contain less than required columns", ErrorCodes::INCORRECT_INDEX); if (block.columns.size() > required_columns.size()) throw Exception("Index contain duplicate columns", ErrorCodes::INCORRECT_INDEX); block.num_columns = block.columns.size(); } }
// ************************************************************************** // Function: LoadParameterList // Purpose: Loads the current list of parameters from a parameter file // It does NOT load system critical dynamic parameters (e.g., ports, // IP addresses) // Parameters: char *filename - filename of the parameterlist // nonexisting - if true, load parameters, even if they currently do // not exist in the list // Returns: true - successful // false - error // ************************************************************************** bool ParamList::Load( const string& inFileName, bool inImportNonexisting ) { ifstream file( inFileName.c_str() ); ParamList paramsFromFile; file >> paramsFromFile; if( file.fail() ) return false; // If desired, exclude parameters missing from the main parameter list. typedef set<string> NameSet; NameSet unwantedParams; if( !inImportNonexisting ) for( int i = 0; i < paramsFromFile.Size(); ++i ) if( !Exists( paramsFromFile[i].Name() ) ) unwantedParams.insert( paramsFromFile[i].Name() ); for( NameSet::const_iterator i = unwantedParams.begin(); i != unwantedParams.end(); ++i ) paramsFromFile.Delete( *i ); for( int i = 0; i < paramsFromFile.Size(); ++i ) { Param &p = paramsFromFile[i], &q = ByName( p.Name() ); if( !q.Readonly() ) q.AssignValues( p, true ); } return true; }
void DatabaseTracker::recordDeletingDatabase(SecurityOrigin *origin, const String& name) { ASSERT(!m_databaseGuard.tryLock()); ASSERT(canDeleteDatabase(origin, name)); NameSet* nameSet = m_beingDeleted.get(origin); if (!nameSet) { nameSet = new NameSet(); m_beingDeleted.set(origin->isolatedCopy(), nameSet); } ASSERT(!nameSet->contains(name)); nameSet->add(name.isolatedCopy()); }
void EnvironmentBase::RangeCheckParams( const ParamList* inParamList, const NameSet& inCheck ) { for( NameSetMap::const_iterator i = OwnedParams().begin(); i != OwnedParams().end(); ++i ) { vector<string> inters( inCheck.size() ); vector<string>::iterator inters_end = set_intersection( inCheck.begin(), inCheck.end(), i->second.begin(), i->second.end(), inters.begin(), Param::NameCmp() ); for( vector<string>::const_iterator j = inters.begin(); j != inters_end; ++j ) { const Param& p = ( *inParamList )[ *j ]; const string& lowRangeStr = p.LowRange(), & highRangeStr = p.HighRange(); bool checkLowRange = ( !lowRangeStr.empty() ), checkHighRange = ( !highRangeStr.empty() ); if( checkLowRange ) { double lowRange = ::atof( lowRangeStr.c_str() ); for( int j = 0; j < p.NumRows(); ++j ) for( int k = 0; k < p.NumColumns(); ++k ) { double value = ::atof( p.Value( j, k ).ToString().c_str() ); if( value < lowRange ) bcierr__ << DescribeValue( p, j, k ) << " is " << value << ", exceeds lower range (" << lowRange << ")"; } } if( checkHighRange ) { double highRange = ::atof( highRangeStr.c_str() ); for( int j = 0; j < p.NumRows(); ++j ) for( int k = 0; k < p.NumColumns(); ++k ) { double value = ::atof( p.Value( j, k ).ToString().c_str() ); if( value > highRange ) bcierr__ << DescribeValue( p, j, k ) << " is " << value << ", exceeds high range (" << highRange << ")"; } } if( checkLowRange || checkHighRange ) ParamsRangeChecked().insert( p.Name() ); } } }
void DatabaseTracker::doneDeletingDatabase(SecurityOrigin *origin, const String& name) { ASSERT(!m_databaseGuard.tryLock()); NameSet* nameSet = m_beingDeleted.get(origin); ASSERT(nameSet); if (!nameSet) return; ASSERT(nameSet->contains(name)); nameSet->remove(name); if (nameSet->isEmpty()) { m_beingDeleted.remove(origin); delete nameSet; } }
void *evalUnique(void *vptr_value) { int start; char *wordArray; NameSet *p = (NameSet *)vptr_value; start = myNameSet->start; while(start < end) { string word = p.at(start); word = substr(0,word.find(' ')); int wordLength = (int)word.length(); int origLength = (int)word.length(); int containsDupe = 0; wordArray = new char [wordLength+1]; strcpy (wordArray, word.c_str()); for(int x=0;x<wordLength;x++) { for(int j=0;j<wordLength;j++) { if(x==j) { continue; } else if(* (wordArray+x)==*(wordArray+j)) { k=j; wordLength--; containsDupe=1; while(k < wordLength) { *(wordArray+k)=*(wordArray+k+1); k++; } j=0; } } } if(containsDupe == 0) { uniqCount++; } if(containsDupe == 0 && origLength >= 6) { //outputFile << "Line: " << lineNumber << " Word: " << word << " Dupe: " << containsDupe << " Unique: " << wordLength << endl; //outputFile << word << endl; uniqCounterArray[lineNumber] = wordLength; } //printf("word read is: %s Dupe: %d Size: %d\n", word, containsDupe,origLength); //strcpy(word,""); delete[] wordArray; start++; } }
bool filterBlockWithQuery(ASTPtr query, Block & block, const Context & context) { query = query->clone(); const ASTSelectQuery & select = typeid_cast<ASTSelectQuery & >(*query); if (!select.where_expression && !select.prewhere_expression) return false; NameSet columns; for (const auto & it : block.getColumnsList()) columns.insert(it.name); /// Составим выражение, вычисляющее выражения в WHERE и PREWHERE, зависящие только от имеющихся столбцов. std::vector<ASTPtr> functions; if (select.where_expression) extractFunctions(select.where_expression, columns, functions); if (select.prewhere_expression) extractFunctions(select.prewhere_expression, columns, functions); ASTPtr expression_ast = buildWhereExpression(functions); if (!expression_ast) return false; /// Распарсим и вычислим выражение. ExpressionAnalyzer analyzer(expression_ast, context, {}, block.getColumnsList()); ExpressionActionsPtr actions = analyzer.getActions(false); actions->execute(block); /// Отфильтруем блок. String filter_column_name = expression_ast->getColumnName(); ColumnPtr filter_column = block.getByName(filter_column_name).column; if (auto converted = filter_column->convertToFullColumnIfConst()) filter_column = converted; const IColumn::Filter & filter = dynamic_cast<ColumnUInt8 &>(*filter_column).getData(); if (std::accumulate(filter.begin(), filter.end(), 0ul) == filter.size()) return false; for (size_t i = 0; i < block.columns(); ++i) { ColumnPtr & column = block.safeGetByPosition(i).column; column = column->filter(filter, -1); } return true; }
/*! TODO: */ TranslucencyModule::TranslucencyModule() :Module(MODULE_NAME, MODULE_DISPLAY_NAME, TRANSLUCENCYMODULE_VERSION), mParamMode(PARAMETER_NAME_MODE, Variable::eVariableTypeString, "Translucency mode", "The type of translucency to perform", false), mParamAlpha(PARAMETER_NAME_ALPHA, Variable::eVariableTypeDouble, "Translucency alpha", "Percentage of translucency (0=opaque, 1=invisible)", true), mParamInvert(PARAMETER_NAME_INVERT, Variable::eVariableTypeBool, "Invert mask", "Invert plain or alpha mask", true) { mShortDescription = "Module for embedding a color image in another one with translucency effect"; mLongDescription = "This module embeds an image into another one using an alpha mask (translucency)"; ValueSet lModeName; lModeName.insert(Value("image", "Whole Image", "Whole image is made translucent using the specified alpha value")); lModeName.insert(Value("plainmask", "Plain Mask", "Only the image pixels where the provided mask is non zero are made translucent using the specified alpha value")); lModeName.insert(Value("alphamask", "Alpha Mask", "Each pixel of the mask specify the alpha value to apply to the corresponding image pixel")); mParamMode.setPossibleValues(lModeName); NameSet lDependentParameterSet; lDependentParameterSet.insert(PARAMETER_NAME_ALPHA); lDependentParameterSet.insert(PARAMETER_NAME_INVERT); mParamMode.setDependentParameterNameSet(lDependentParameterSet); mParamMode.setValueStr("plainmask"); mParamAlpha.setValue(0.5); mParamAlpha.setMinValue("0"); mParamAlpha.setMaxValue("1"); mParamInvert.setValue(false); newParameter(mParamMode); newParameter(mParamAlpha); newParameter(mParamInvert); mInputSlotBackgroundColorImage = newSlot(new ModuleSlot(this, INPUT_SLOT_NAME_BGIMAGE, INPUT_SLOT_DISPLAYNAME_BGIMAGE, "Background image on which embedding with be performed")); mInputSlotTranslucencyColorImage = newSlot(new ModuleSlot(this, INPUT_SLOT_NAME_TRANSIMAGE, INPUT_SLOT_DISPLAYNAME_TRANSIMAGE, "Image that is going to be made translucent")); mInputSlotTranslucencyMaskImage = newSlot(new ModuleSlot(this, INPUT_SLOT_NAME_ALPHAMASK, INPUT_SLOT_DISPLAYNAME_ALPHAMASK, "Alpha mask used to make translucent image")); mOutputSlot = newSlot(new ModuleSlot(this, OUTPUT_SLOT_NAME_IMAGE, OUTPUT_SLOT_DISPLAYNAME_IMAGE, "Output image", &mOutputFrame)); mOutputFrameIpl = NULL; mOutputFrame = NULL; mTmpFrame1 = NULL; mTmpFrame2 = NULL; mTmpFrame3 = NULL; mTmpFrame4 = NULL; mTmpFrame5 = NULL; }
NamesAndTypesList NamesAndTypesList::filter(const NameSet & names) const { NamesAndTypesList res; for (const NameAndTypePair & column : *this) { if (names.count(column.name)) res.push_back(column); } return res; }
bool filterBlockWithQuery(const ASTPtr & query, Block & block, const Context & context) { const ASTSelectQuery & select = typeid_cast<const ASTSelectQuery & >(*query); if (!select.where_expression && !select.prewhere_expression) return false; NameSet columns; for (const auto & it : block.getColumnsList()) columns.insert(it.name); /// We will create an expression that evaluates the expressions in WHERE and PREWHERE, depending only on the existing columns. std::vector<ASTPtr> functions; if (select.where_expression) extractFunctions(select.where_expression, columns, functions); if (select.prewhere_expression) extractFunctions(select.prewhere_expression, columns, functions); ASTPtr expression_ast = buildWhereExpression(functions); if (!expression_ast) return false; /// Let's parse and calculate the expression. ExpressionAnalyzer analyzer(expression_ast, context, {}, block.getColumnsList()); ExpressionActionsPtr actions = analyzer.getActions(false); actions->execute(block); /// Filter the block. String filter_column_name = expression_ast->getColumnName(); ColumnPtr filter_column = block.getByName(filter_column_name).column; if (auto converted = filter_column->convertToFullColumnIfConst()) filter_column = converted; const IColumn::Filter & filter = dynamic_cast<ColumnUInt8 &>(*filter_column).getData(); if (countBytesInFilter(filter) == 0) return false; for (size_t i = 0; i < block.columns(); ++i) { ColumnPtr & column = block.safeGetByPosition(i).column; column = column->filter(filter, -1); } return true; }
/// Verifying that the function depends only on the specified columns static bool isValidFunction(ASTPtr expression, const NameSet & columns) { for (size_t i = 0; i < expression->children.size(); ++i) if (!isValidFunction(expression->children[i], columns)) return false; if (const ASTIdentifier * identifier = typeid_cast<const ASTIdentifier *>(&*expression)) { if (identifier->kind == ASTIdentifier::Kind::Column) return columns.count(identifier->name); } return true; }
void AnimationCleanerVisitor::cleanInvalidUpdateMorph() { // Removes unused UpdateMorph targets (i.e. name does not match any MorphGeometry target) for(AnimationUpdateCallBackMap::iterator update = _updates.begin() ; update != _updates.end() ; ++ update) { osgAnimation::UpdateMorph *updateMorph = dynamic_cast<osgAnimation::UpdateMorph*>(update->first.get()); if(!updateMorph) continue; NameSet toRemove; for(unsigned int i = 0, numTarget = updateMorph->getNumTarget(); i < numTarget; ++i) { const std::string& name = updateMorph->getTargetName(i); if(_morphTargets.count(name) == 0) { toRemove.insert(name); } } for(NameSet::iterator targetName = toRemove.begin(); targetName != toRemove.end(); ++targetName) { updateMorph->removeTarget(*targetName); } } // Removes empty UpdateMorphCallback for(AnimationUpdateCallBackMap::iterator update = _updates.begin() ; update != _updates.end() ; ) { osgAnimation::UpdateMorph *updateMorph = dynamic_cast<osgAnimation::UpdateMorph*>(update->first.get()); if(!updateMorph || updateMorph->getNumTarget() != 0) { ++ update; } else { osg::Callback *callBack = update->second.get()->getUpdateCallback(); if(callBack) { if(callBack == updateMorph) update->second.get()->setUpdateCallback(callBack->getNestedCallback()); else callBack->removeNestedCallback(updateMorph); } _updates.erase(update ++); } } }
// ************************************************************************** // Function: LoadParameterList // Purpose: Loads the current list of parameters from a parameter file // It does NOT load system critical dynamic parameters (e.g., ports, // IP addresses) // Parameters: char *filename - filename of the parameterlist // nonexisting - if true, load parameters, even if they currently do // not exist in the list // Returns: true - successful // false - error // ************************************************************************** bool ParamList::Load( const string& inFileName, bool inImportNonexisting ) { ifstream file( inFileName.c_str() ); ParamList paramsFromFile; file >> paramsFromFile; if( file.fail() ) return false; typedef set<string> NameSet; NameSet unwantedParams; // Exclude parameters from unwanted sections. const char* unwantedSections[] = { "System", }; for( size_t j = 0; j < sizeof( unwantedSections ) / sizeof( *unwantedSections ); ++j ) for( ParamContainer::const_iterator i = paramsFromFile.mParams.begin(); i != paramsFromFile.mParams.end(); ++i ) if( Param::strciequal( i->Param.Section(), unwantedSections[ j ] ) ) unwantedParams.insert( i->Param.mName ); // If desired, exclude parameters missing from the main parameter list. if( !inImportNonexisting ) for( ParamContainer::const_iterator i = paramsFromFile.mParams.begin(); i != paramsFromFile.mParams.end(); ++i ) if( mNameIndex.find( i->Param.mName ) == mNameIndex.end() ) unwantedParams.insert( i->Param.mName ); for( NameSet::const_iterator i = unwantedParams.begin(); i != unwantedParams.end(); ++i ) paramsFromFile.Delete( *i ); for( ParamContainer::const_iterator i = paramsFromFile.mParams.begin(); i != paramsFromFile.mParams.end(); ++i ) ( *this )[ i->Param.mName ].AssignValues( i->Param ); return true; }
static void mergeKeySets(NameSet *dest, const NameSet &src) { NameSet::const_iterator it = src.constBegin(); for (; it != src.constEnd(); ++it) dest->insert(unescapedKey(it.key()), QString()); }
/*! \todo */ void Parameter::setDependentParameterNameSet(const NameSet& inNameSet) throw() { mDependentParameterNameSet.clear(); for(NameSet::const_iterator lItr = inNameSet.begin(); lItr != inNameSet.end(); lItr++) mDependentParameterNameSet.insert(lItr->c_str()); }
bool DatabaseTracker::isDeletingDatabase(SecurityOrigin *origin, const String& name) { ASSERT(!m_databaseGuard.tryLock()); NameSet* nameSet = m_beingDeleted.get(origin); return nameSet && nameSet->contains(name); }
void WidgetDataBase::loadPlugins() { typedef QMap<QString, int> NameIndexMap; typedef QList<QDesignerWidgetDataBaseItemInterface*> ItemList; typedef QMap<QString, QDesignerWidgetDataBaseItemInterface*> NameItemMap; typedef QSet<QString> NameSet; // 1) create a map of existing custom classes NameIndexMap existingCustomClasses; NameSet nonCustomClasses; const int count = m_items.size(); for (int i = 0; i < count; i++) { const QDesignerWidgetDataBaseItemInterface* item = m_items[i]; if (item->isCustom() && !item->isPromoted()) existingCustomClasses.insert(item->name(), i); else nonCustomClasses.insert(item->name()); } // 2) create a list plugins ItemList pluginList; const QDesignerPluginManager *pm = m_core->pluginManager(); foreach(QDesignerCustomWidgetInterface* c, pm->registeredCustomWidgets()) pluginList += createCustomWidgetItem(c, pm->customWidgetData(c)); // 3) replace custom classes or add new ones, remove them from existingCustomClasses, // leaving behind deleted items unsigned replacedPlugins = 0; unsigned addedPlugins = 0; unsigned removedPlugins = 0; if (!pluginList.empty()) { ItemList::const_iterator cend = pluginList.constEnd(); for (ItemList::const_iterator it = pluginList.constBegin();it != cend; ++it ) { QDesignerWidgetDataBaseItemInterface* pluginItem = *it; const QString pluginName = pluginItem->name(); NameIndexMap::iterator existingIt = existingCustomClasses.find(pluginName); if (existingIt == existingCustomClasses.end()) { // Add new class. if (nonCustomClasses.contains(pluginName)) { designerWarning(tr("A custom widget plugin whose class name (%1) matches that of an existing class has been found.").arg(pluginName)); } else { append(pluginItem); addedPlugins++; } } else { // replace existing info const int existingIndex = existingIt.value(); delete m_items[existingIndex]; m_items[existingIndex] = pluginItem; existingCustomClasses.erase(existingIt); replacedPlugins++; } } } // 4) remove classes that have not been matched. The stored indexes become invalid while deleting. if (!existingCustomClasses.empty()) { NameIndexMap::const_iterator cend = existingCustomClasses.constEnd(); for (NameIndexMap::const_iterator it = existingCustomClasses.constBegin();it != cend; ++it ) { const int index = indexOfClassName(it.key()); if (index != -1) { remove(index); removedPlugins++; } } } if (debugWidgetDataBase) qDebug() << "WidgetDataBase::loadPlugins(): " << addedPlugins << " added, " << replacedPlugins << " replaced, " << removedPlugins << "deleted."; }
bool MergeTreeBlockInputStream::getNewTask() try { /// Produce only one task if (!is_first_task) { finish(); return false; } is_first_task = false; Names pre_column_names, column_names = ordered_names; bool remove_prewhere_column = false; /// inject columns required for defaults evaluation bool should_reorder = !injectRequiredColumns(storage, data_part, column_names).empty(); if (prewhere_actions) { pre_column_names = prewhere_actions->getRequiredColumns(); if (pre_column_names.empty()) pre_column_names.push_back(column_names[0]); const auto injected_pre_columns = injectRequiredColumns(storage, data_part, pre_column_names); if (!injected_pre_columns.empty()) should_reorder = true; const NameSet pre_name_set(pre_column_names.begin(), pre_column_names.end()); /// If the expression in PREWHERE is not a column of the table, you do not need to output a column with it /// (from storage expect to receive only the columns of the table). remove_prewhere_column = !pre_name_set.count(prewhere_column); Names post_column_names; for (const auto & name : column_names) if (!pre_name_set.count(name)) post_column_names.push_back(name); column_names = post_column_names; } /// will be used to distinguish between PREWHERE and WHERE columns when applying filter column_name_set = NameSet{column_names.begin(), column_names.end()}; if (check_columns) { /// Under owned_data_part->columns_lock we check that all requested columns are of the same type as in the table. /// This may be not true in case of ALTER MODIFY. if (!pre_column_names.empty()) storage.check(data_part->columns, pre_column_names); if (!column_names.empty()) storage.check(data_part->columns, column_names); pre_columns = storage.getColumnsList().addTypes(pre_column_names); columns = storage.getColumnsList().addTypes(column_names); } else { pre_columns = data_part->columns.addTypes(pre_column_names); columns = data_part->columns.addTypes(column_names); } /** @note you could simply swap `reverse` in if and else branches of MergeTreeDataSelectExecutor, * and remove this reverse. */ MarkRanges remaining_mark_ranges = all_mark_ranges; std::reverse(remaining_mark_ranges.begin(), remaining_mark_ranges.end()); auto size_predictor = (preferred_block_size_bytes == 0) ? nullptr : std::make_unique<MergeTreeBlockSizePredictor>(data_part, ordered_names, data_part->storage.getSampleBlock()); task = std::make_unique<MergeTreeReadTask>(data_part, remaining_mark_ranges, part_index_in_query, ordered_names, column_name_set, columns, pre_columns, remove_prewhere_column, should_reorder, std::move(size_predictor)); if (!reader) { if (use_uncompressed_cache) owned_uncompressed_cache = storage.context.getUncompressedCache(); owned_mark_cache = storage.context.getMarkCache(); reader = std::make_unique<MergeTreeReader>( path, data_part, columns, owned_uncompressed_cache.get(), owned_mark_cache.get(), save_marks_in_cache, storage, all_mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size); if (prewhere_actions) pre_reader = std::make_unique<MergeTreeReader>( path, data_part, pre_columns, owned_uncompressed_cache.get(), owned_mark_cache.get(), save_marks_in_cache, storage, all_mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size); } return true; } catch (...) { /// Suspicion of the broken part. A part is added to the queue for verification. if (getCurrentExceptionCode() != ErrorCodes::MEMORY_LIMIT_EXCEEDED) storage.reportBrokenPart(data_part->name); throw; }
std::vector<std::size_t> MergeTreeReadPool::fillPerPartInfo( RangesInDataParts & parts, const ExpressionActionsPtr & prewhere_actions, const String & prewhere_column_name, const bool check_columns) { std::vector<std::size_t> per_part_sum_marks; for (const auto i : ext::range(0, parts.size())) { auto & part = parts[i]; /// Read marks for every data part. size_t sum_marks = 0; /// Ranges are in right-to-left order, due to 'reverse' in MergeTreeDataSelectExecutor. for (const auto & range : part.ranges) sum_marks += range.end - range.begin; per_part_sum_marks.push_back(sum_marks); per_part_columns_lock.push_back(std::make_unique<Poco::ScopedReadRWLock>( part.data_part->columns_lock)); /// inject column names required for DEFAULT evaluation in current part auto required_column_names = column_names; const auto injected_columns = injectRequiredColumns(part.data_part, required_column_names); auto should_reoder = !injected_columns.empty(); Names required_pre_column_names; if (prewhere_actions) { /// collect columns required for PREWHERE evaluation required_pre_column_names = prewhere_actions->getRequiredColumns(); /// there must be at least one column required for PREWHERE if (required_pre_column_names.empty()) required_pre_column_names.push_back(required_column_names[0]); /// PREWHERE columns may require some additional columns for DEFAULT evaluation const auto injected_pre_columns = injectRequiredColumns(part.data_part, required_pre_column_names); if (!injected_pre_columns.empty()) should_reoder = true; /// will be used to distinguish between PREWHERE and WHERE columns when applying filter const NameSet pre_name_set{ std::begin(required_pre_column_names), std::end(required_pre_column_names) }; /** If expression in PREWHERE is not table column, then no need to return column with it to caller * (because storage is expected only to read table columns). */ per_part_remove_prewhere_column.push_back(0 == pre_name_set.count(prewhere_column_name)); Names post_column_names; for (const auto & name : required_column_names) if (!pre_name_set.count(name)) post_column_names.push_back(name); required_column_names = post_column_names; } else per_part_remove_prewhere_column.push_back(false); per_part_column_name_set.emplace_back(std::begin(required_column_names), std::end(required_column_names)); if (check_columns) { /** Under part->columns_lock check that all requested columns in part are of same type that in table. * This could be violated during ALTER MODIFY. */ if (!required_pre_column_names.empty()) data.check(part.data_part->columns, required_pre_column_names); if (!required_column_names.empty()) data.check(part.data_part->columns, required_column_names); per_part_pre_columns.push_back(data.getColumnsList().addTypes(required_pre_column_names)); per_part_columns.push_back(data.getColumnsList().addTypes(required_column_names)); } else { per_part_pre_columns.push_back(part.data_part->columns.addTypes(required_pre_column_names)); per_part_columns.push_back(part.data_part->columns.addTypes(required_column_names)); } per_part_should_reorder.push_back(should_reoder); this->parts.push_back({ part.data_part, part.part_index_in_query }); } return per_part_sum_marks; }
void Join::joinBlockImpl( Block & block, const Names & key_names_left, const NameSet & needed_key_names_right, const Block & block_with_columns_to_add, const Maps & maps_) const { size_t keys_size = key_names_left.size(); ColumnRawPtrs key_columns(keys_size); /// Rare case, when keys are constant. To avoid code bloat, simply materialize them. Columns materialized_columns; materialized_columns.reserve(keys_size); /// Memoize key columns to work with. for (size_t i = 0; i < keys_size; ++i) { materialized_columns.emplace_back(recursiveRemoveLowCardinality(block.getByName(key_names_left[i]).column->convertToFullColumnIfConst())); key_columns[i] = materialized_columns.back().get(); } /// Keys with NULL value in any column won't join to anything. ColumnPtr null_map_holder; ConstNullMapPtr null_map{}; extractNestedColumnsAndNullMap(key_columns, null_map_holder, null_map); size_t existing_columns = block.columns(); /** If you use FULL or RIGHT JOIN, then the columns from the "left" table must be materialized. * Because if they are constants, then in the "not joined" rows, they may have different values * - default values, which can differ from the values of these constants. */ if (getFullness(kind)) { for (size_t i = 0; i < existing_columns; ++i) { block.getByPosition(i).column = block.getByPosition(i).column->convertToFullColumnIfConst(); /// If use_nulls, convert left columns (except keys) to Nullable. if (use_nulls) { if (std::end(key_names_left) == std::find(key_names_left.begin(), key_names_left.end(), block.getByPosition(i).name)) convertColumnToNullable(block.getByPosition(i)); } } } /** For LEFT/INNER JOIN, the saved blocks do not contain keys. * For FULL/RIGHT JOIN, the saved blocks contain keys; * but they will not be used at this stage of joining (and will be in `AdderNonJoined`), and they need to be skipped. */ size_t num_columns_to_skip = 0; if (getFullness(kind)) num_columns_to_skip = keys_size; /// Add new columns to the block. size_t num_columns_to_add = sample_block_with_columns_to_add.columns(); MutableColumns added_columns; added_columns.reserve(num_columns_to_add); std::vector<std::pair<decltype(ColumnWithTypeAndName::type), decltype(ColumnWithTypeAndName::name)>> added_type_name; added_type_name.reserve(num_columns_to_add); std::vector<size_t> right_indexes; right_indexes.reserve(num_columns_to_add); for (size_t i = 0; i < num_columns_to_add; ++i) { const ColumnWithTypeAndName & src_column = sample_block_with_columns_to_add.safeGetByPosition(i); /// Don't insert column if it's in left block or not explicitly required. if (!block.has(src_column.name) && block_with_columns_to_add.has(src_column.name)) { added_columns.push_back(src_column.column->cloneEmpty()); added_columns.back()->reserve(src_column.column->size()); added_type_name.emplace_back(src_column.type, src_column.name); right_indexes.push_back(num_columns_to_skip + i); } } size_t rows = block.rows(); std::unique_ptr<IColumn::Filter> filter; bool filter_left_keys = (kind == ASTTableJoin::Kind::Inner || kind == ASTTableJoin::Kind::Right) && strictness == ASTTableJoin::Strictness::Any; filter = std::make_unique<IColumn::Filter>(rows); /// Used with ALL ... JOIN IColumn::Offset current_offset = 0; std::unique_ptr<IColumn::Offsets> offsets_to_replicate; if (strictness == ASTTableJoin::Strictness::All) offsets_to_replicate = std::make_unique<IColumn::Offsets>(rows); switch (type) { #define M(TYPE) \ case Join::Type::TYPE: \ joinBlockImplType<KIND, STRICTNESS, typename KeyGetterForType<Join::Type::TYPE>::Type>(\ *maps_.TYPE, rows, key_columns, key_sizes, added_columns, null_map, \ filter, current_offset, offsets_to_replicate, right_indexes); \ break; APPLY_FOR_JOIN_VARIANTS(M) #undef M default: throw Exception("Unknown JOIN keys variant.", ErrorCodes::UNKNOWN_SET_DATA_VARIANT); } const auto added_columns_size = added_columns.size(); for (size_t i = 0; i < added_columns_size; ++i) block.insert(ColumnWithTypeAndName(std::move(added_columns[i]), added_type_name[i].first, added_type_name[i].second)); /// If ANY INNER | RIGHT JOIN - filter all the columns except the new ones. if (filter_left_keys) for (size_t i = 0; i < existing_columns; ++i) block.safeGetByPosition(i).column = block.safeGetByPosition(i).column->filter(*filter, -1); ColumnUInt64::Ptr mapping; /// Add join key columns from right block if they has different name. for (size_t i = 0; i < key_names_right.size(); ++i) { auto & right_name = key_names_right[i]; auto & left_name = key_names_left[i]; if (needed_key_names_right.count(right_name) && !block.has(right_name)) { const auto & col = block.getByName(left_name); auto column = col.column; if (!filter_left_keys) { if (!mapping) { auto mut_mapping = ColumnUInt64::create(column->size()); auto & data = mut_mapping->getData(); size_t size = column->size(); for (size_t j = 0; j < size; ++j) data[j] = (*filter)[j] ? j : size; mapping = std::move(mut_mapping); } auto mut_column = (*std::move(column)).mutate(); mut_column->insertDefault(); column = mut_column->index(*mapping, 0); } block.insert({column, col.type, right_name}); } } /// If ALL ... JOIN - we replicate all the columns except the new ones. if (offsets_to_replicate) { for (size_t i = 0; i < existing_columns; ++i) block.safeGetByPosition(i).column = block.safeGetByPosition(i).column->replicate(*offsets_to_replicate); } }