Beispiel #1
0
const indri::infnet::InferenceNetwork::MAllResults& indri::infnet::InferenceNetwork::evaluate() {
  // count this query occurrence
  _repository.countQuery();

  // fetch the current index state
  indri::collection::Repository::index_state indexes = _repository.indexes();
  
  for( size_t i=0; i<indexes->size(); i++ ) {
    indri::index::Index& index = *(*indexes)[i];
    indri::thread::ScopedLock iterators( index.iteratorLock() );

    indri::thread::ScopedLock statistics( index.statisticsLock() );
    _indexChanged( index );
    statistics.unlock();

    // evaluate query against the index
    _evaluateIndex( index );

    // remove all the iterators
    _indexFinished( index );
  }

  _results.clear();
  for( size_t i=0; i<_evaluators.size(); i++ ) {
    _results[ _evaluators[i]->getName() ] = _evaluators[i]->getResults();
  }

  return _results;
}
    static void NO_INLINE execute(Map ** source_maps, size_t num_maps, Map *& result_map,
                        Merger && merger,
                        ThreadPool &)
    {
        std::vector<typename Map::iterator> iterators(num_maps);
        for (size_t i = 1; i < num_maps; ++i)
            iterators[i] = source_maps[i]->begin();

        result_map = source_maps[0];

        while (true)
        {
            bool finish = true;
            for (size_t i = 1; i < num_maps; ++i)
            {
                if (iterators[i] == source_maps[i]->end())
                    continue;

                finish = false;
                merger((*result_map)[iterators[i]->first], iterators[i]->second);
                ++iterators[i];
            }

            if (finish)
                break;
        }
    }
void ObjectsToCostmap::setCostInPolygon(const grid_map::Polygon& polygon, const std::string& gridmap_layer_name,
                                       const float score, grid_map::GridMap& objects_costmap)
{
  grid_map::PolygonIterator iterators(objects_costmap, polygon);
  for (grid_map::PolygonIterator iterator(objects_costmap, polygon); !iterator.isPastEnd(); ++iterator)
  {
    const float current_score = objects_costmap.at(gridmap_layer_name, *iterator);
    if (score > current_score)
    {
      objects_costmap.at(gridmap_layer_name, *iterator) = score;
    }
  }
}
PhaseDetector::PhaseDetector(std::string input_filename, unsigned int max_phase_sz, unsigned int elems_to_read) : 
	max_phase_size(max_phase_sz), 
	max_elems(elems_to_read),
	has_new_phase(false)
{
	std::ifstream input_file(input_filename);
	std::vector<int64_t> line_values;
	std::string input_str;

	if(input_file)
	{
		///reading the number of entries and the number of iterators
		std::getline(input_file, input_str, ' ');
		entries_num = std::stol(input_str);
		std::getline(input_file, input_str);
		iterator_num = std::stol(input_str);

		///reading the iterators' bounds	
		for(unsigned int i = 0; i < iterator_num - 1; i++)
		{
			std::getline(input_file, input_str, ' ');
			bounds.insert(bounds.begin(),std::stol(input_str));
		}
		std::getline(input_file, input_str);
		bounds.insert(bounds.begin(), std::stol(input_str));
		
		///reading the memory positions and their corresponding iteration vectors
		for(unsigned int i = 0; i < max_elems; i++)
		{
			std::getline(input_file, input_str,'\n');
			if(input_file.eof()) break;

			line_values = splitEntry(input_str);

			std::vector<int64_t> iterators(line_values.begin() + 1, line_values.end());
			addr_vector.push_back(line_values[0]);
			iterators_vector.push_back(iterators);
		}
		next_start = 0;
	}
	else
	{
		cerr << "Couldn't open the input file\n";
		exit(EXIT_FAILURE);
	}
	input_file.close();	
}
Beispiel #5
0
int main() {
    initWithSingleElement();
    initWithContainers();
    initWithIterators();
    copyAndMove();
    appendAndJoin();
    iterators();
    modify();
    operations();
    compare();
    assignment();
    emplace();
    merge();
    sort();
    resize();
    
}
        virtual bool run(OperationContext* txn, const string& dbname, BSONObj& cmdObj, int options,
                          string& errmsg, BSONObjBuilder& result,
                          bool fromRepl = false ) {

            NamespaceString ns( dbname, cmdObj[name].String() );

            AutoGetCollectionForRead ctx(txn, ns.ns());

            Collection* collection = ctx.getCollection();
            if ( !collection )
                return appendCommandStatus( result,
                                            Status( ErrorCodes::NamespaceNotFound,
                                                    str::stream() <<
                                                    "ns does not exist: " << ns.ns() ) );

            size_t numCursors = static_cast<size_t>( cmdObj["numCursors"].numberInt() );

            if ( numCursors == 0 || numCursors > 10000 )
                return appendCommandStatus( result,
                                            Status( ErrorCodes::BadValue,
                                                    str::stream() <<
                                                    "numCursors has to be between 1 and 10000" <<
                                                    " was: " << numCursors ) );

            OwnedPointerVector<RecordIterator> iterators(collection->getManyIterators(txn));

            if (iterators.size() < numCursors) {
                numCursors = iterators.size();
            }

            OwnedPointerVector<PlanExecutor> execs;
            for ( size_t i = 0; i < numCursors; i++ ) {
                WorkingSet* ws = new WorkingSet();
                MultiIteratorStage* mis = new MultiIteratorStage(txn, ws, collection);
                // Takes ownership of 'ws' and 'mis'.
                auto_ptr<PlanExecutor> curExec(new PlanExecutor(txn, ws, mis, collection));

                // Each of the plan executors should yield automatically. We pass "false" to
                // indicate that 'curExec' should not register itself, as it will get registered
                // by ClientCursor instead.
                curExec->setYieldPolicy(PlanExecutor::YIELD_AUTO, false);

                // Need to save state while yielding locks between now and newGetMore.
                curExec->saveState();

                execs.push_back(curExec.release());
            }

            // transfer iterators to executors using a round-robin distribution.
            // TODO consider using a common work queue once invalidation issues go away.
            for (size_t i = 0; i < iterators.size(); i++) {
                PlanExecutor* theExec = execs[i % execs.size()];
                MultiIteratorStage* mis = static_cast<MultiIteratorStage*>(theExec->getRootStage());
                mis->addIterator(iterators.releaseAt(i));
            }

            {
                BSONArrayBuilder bucketsBuilder;
                for (size_t i = 0; i < execs.size(); i++) {
                    // transfer ownership of an executor to the ClientCursor (which manages its own
                    // lifetime).
                    ClientCursor* cc = new ClientCursor( collection, execs.releaseAt(i) );

                    // we are mimicking the aggregation cursor output here
                    // that is why there are ns, ok and empty firstBatch
                    BSONObjBuilder threadResult;
                    {
                        BSONObjBuilder cursor;
                        cursor.appendArray( "firstBatch", BSONObj() );
                        cursor.append( "ns", ns );
                        cursor.append( "id", cc->cursorid() );
                        threadResult.append( "cursor", cursor.obj() );
                    }
                    threadResult.appendBool( "ok", 1 );

                    bucketsBuilder.append( threadResult.obj() );
                }
                result.appendArray( "cursors", bucketsBuilder.obj() );
            }

            return true;

        }
/*
 * Merge the runs from the input file into the output file.
 */
bool MergeSorter::merge_pass()
{
	// Check how many runs we're merging.
	const unsigned int RECORDS_PER_PAGE = records_per_page(_page_size);
	int run_count = (_records_pages + (_run_length - 1)) / _run_length;

	// Merge runs while there are runs left.
	int run_index;
	for (run_index = 0; run_index < run_count; run_index += _k) {
		// Check how many we're merging now.
		int merge_count = run_count - run_index;
		if (merge_count > _k) {
			merge_count = _k;
		}
		std::vector<RunIterator> iterators(merge_count);

		// Set up the iterators.
		int i;
		for (i = 0; i < merge_count; ++i) {
			int current_index = run_index + i;
			int run_page_index = current_index * _run_length;
			int run_offset = current_index * _run_length * _page_size;
			int page_buffer_offset = ((i + 1) * _page_size);
			RunIterator& iter = iterators[i];
			iter.set_buffer(_pages + page_buffer_offset, _page_size);

			// Calculate how many pages this run has.
			int length = _records_pages - run_page_index;
			if (length > _run_length) {
				length = _run_length;
			}
			iter.set_run(_run_input, run_offset, length);

			// Move to first value.
			if (!iter.next()) {
				return false;
			}
		}

		// We'll remove iterators when we're done with them, so loop until we have no more.
		// Use the first page as the output buffer.
		Record *output_buffer = reinterpret_cast<Record*>(_pages);
		unsigned int output_record_index = 0;
		while (!iterators.empty()) {
			// Find the minimum current value.
			Record *smallest_record = NULL;
			std::vector<RunIterator>::iterator smallest_j;
			std::vector<RunIterator>::iterator j;
			for (j = iterators.begin(); j != iterators.end(); ++j) {
				RunIterator& iter = *j;

				// Compare this record to current lowest.
				Record *record = iter.current();
				if ((smallest_record == NULL) || (record_compare(record, smallest_record) < 0)) {
					smallest_record = record;
					smallest_j = j;
				}
			}

			// Write the smallest record to merge page.
			memcpy(&output_buffer[output_record_index], smallest_record, RECORD_SIZE);
			++output_record_index;

			// Push smallest iterator to next value.
			RunIterator& smallest_run = *smallest_j;
			if (!smallest_run.next()) {
				return false;
			}
	
			// Remove this iterator if it's empty.
			if (smallest_run.current() == NULL) {
				iterators.erase(smallest_j);
			}

			bool is_done = iterators.empty();
			bool is_full = output_record_index == RECORDS_PER_PAGE;

			// Null out the end entry if page isn't full.
			if (is_done && !is_full) {
				Record *last = &output_buffer[output_record_index];
				(*last)[RECORD_SIZE - 1] = '\0';
			}
			// Write the last page if full or last record.
			else if (is_full || is_done) {
				size_t written = fwrite(output_buffer, _page_size, 1, _run_output);
				if (written != 1) {
					fprintf(stderr, "Failed to write output buffer page to disk.\n");
					return false;
				}
				output_record_index = 0;
			}
		}
	}

	// We merged merge_maximum at a time, so runs are that many times longer.
	_run_length *= _k;
	printf("Runs are length %d.\n", _run_length);
	return true;
}
        virtual bool run( const string& dbname, BSONObj& cmdObj, int options,
                          string& errmsg, BSONObjBuilder& result,
                          bool fromRepl = false ) {

            NamespaceString ns( dbname, cmdObj[name].String() );

            Client::ReadContext ctx(ns.ns());

            Database* db = ctx.ctx().db();
            Collection* collection = db->getCollection( ns );

            if ( !collection )
                return appendCommandStatus( result,
                                            Status( ErrorCodes::NamespaceNotFound,
                                                    str::stream() <<
                                                    "ns does not exist: " << ns.ns() ) );

            size_t numCursors = static_cast<size_t>( cmdObj["numCursors"].numberInt() );

            if ( numCursors == 0 || numCursors > 10000 )
                return appendCommandStatus( result,
                                            Status( ErrorCodes::BadValue,
                                                    str::stream() <<
                                                    "numCursors has to be between 1 and 10000" <<
                                                    " was: " << numCursors ) );

            OwnedPointerVector<RecordIterator> iterators(collection->getManyIterators());

            if (iterators.size() < numCursors) {
                numCursors = iterators.size();
            }

            OwnedPointerVector<MultiIteratorRunner> runners;
            for ( size_t i = 0; i < numCursors; i++ ) {
                runners.push_back(new MultiIteratorRunner(ns.ns(), collection));
            }

            // transfer iterators to runners using a round-robin distribution.
            // TODO consider using a common work queue once invalidation issues go away.
            for (size_t i = 0; i < iterators.size(); i++) {
                runners[i % runners.size()]->addIterator(iterators.releaseAt(i));
            }

            {
                BSONArrayBuilder bucketsBuilder;
                for (size_t i = 0; i < runners.size(); i++) {
                    // transfer ownership of a runner to the ClientCursor (which manages its own
                    // lifetime).
                    ClientCursor* cc = new ClientCursor( collection, runners.releaseAt(i) );

                    // we are mimicking the aggregation cursor output here
                    // that is why there are ns, ok and empty firstBatch
                    BSONObjBuilder threadResult;
                    {
                        BSONObjBuilder cursor;
                        cursor.appendArray( "firstBatch", BSONObj() );
                        cursor.append( "ns", ns );
                        cursor.append( "id", cc->cursorid() );
                        threadResult.append( "cursor", cursor.obj() );
                    }
                    threadResult.appendBool( "ok", 1 );

                    bucketsBuilder.append( threadResult.obj() );
                }
                result.appendArray( "cursors", bucketsBuilder.obj() );
            }

            return true;

        }
int main(int argc, char ** argv)
{
	size_t n = atoi(argv[1]);
	size_t num_threads = atoi(argv[2]);
	size_t method = argc <= 3 ? 0 : atoi(argv[3]);

	std::cerr << std::fixed << std::setprecision(2);

	ThreadPool pool(num_threads);

	Source data(n);

	{
		Stopwatch watch;
		DB::ReadBufferFromFileDescriptor in1(STDIN_FILENO);
		DB::CompressedReadBuffer in2(in1);

		in2.readStrict(reinterpret_cast<char*>(&data[0]), sizeof(data[0]) * n);

		watch.stop();
		std::cerr << std::fixed << std::setprecision(2)
			<< "Vector. Size: " << n
			<< ", elapsed: " << watch.elapsedSeconds()
			<< " (" << n / watch.elapsedSeconds() << " elem/sec.)"
			<< std::endl << std::endl;
	}

	if (!method || method == 1)
	{
		/** Вариант 1.
		  * В разных потоках агрегируем независимо в разные хэш-таблицы.
		  * Затем сливаем их вместе.
		  */

		std::vector<Map> maps(num_threads);

		Stopwatch watch;

		for (size_t i = 0; i < num_threads; ++i)
			pool.schedule(std::bind(aggregate1,
				std::ref(maps[i]),
				data.begin() + (data.size() * i) / num_threads,
				data.begin() + (data.size() * (i + 1)) / num_threads));

		pool.wait();

		watch.stop();
		double time_aggregated = watch.elapsedSeconds();
		std::cerr
			<< "Aggregated in " << time_aggregated
			<< " (" << n / time_aggregated << " elem/sec.)"
			<< std::endl;

		size_t size_before_merge = 0;
		std::cerr << "Sizes: ";
		for (size_t i = 0; i < num_threads; ++i)
		{
			std::cerr << (i == 0 ? "" : ", ") << maps[i].size();
			size_before_merge += maps[i].size();
		}
		std::cerr << std::endl;

		watch.restart();

		for (size_t i = 1; i < num_threads; ++i)
			for (auto it = maps[i].begin(); it != maps[i].end(); ++it)
				maps[0][it->first] += it->second;

		watch.stop();
		double time_merged = watch.elapsedSeconds();
		std::cerr
			<< "Merged in " << time_merged
			<< " (" << size_before_merge / time_merged << " elem/sec.)"
			<< std::endl;

		double time_total = time_aggregated + time_merged;
		std::cerr
			<< "Total in " << time_total
			<< " (" << n / time_total << " elem/sec.)"
			<< std::endl;
		std::cerr << "Size: " << maps[0].size() << std::endl << std::endl;
	}

	if (!method || method == 12)
	{
		/** То же самое, но с оптимизацией для подряд идущих одинаковых значений.
		  */

		std::vector<Map> maps(num_threads);

		Stopwatch watch;

		for (size_t i = 0; i < num_threads; ++i)
			pool.schedule(std::bind(aggregate12,
									std::ref(maps[i]),
									data.begin() + (data.size() * i) / num_threads,
									data.begin() + (data.size() * (i + 1)) / num_threads));

		pool.wait();

		watch.stop();
		double time_aggregated = watch.elapsedSeconds();
		std::cerr
		<< "Aggregated in " << time_aggregated
		<< " (" << n / time_aggregated << " elem/sec.)"
		<< std::endl;

		size_t size_before_merge = 0;
		std::cerr << "Sizes: ";
		for (size_t i = 0; i < num_threads; ++i)
		{
			std::cerr << (i == 0 ? "" : ", ") << maps[i].size();
			size_before_merge += maps[i].size();
		}
		std::cerr << std::endl;

		watch.restart();

		for (size_t i = 1; i < num_threads; ++i)
			for (auto it = maps[i].begin(); it != maps[i].end(); ++it)
				maps[0][it->first] += it->second;

		watch.stop();

		double time_merged = watch.elapsedSeconds();
		std::cerr
		<< "Merged in " << time_merged
		<< " (" << size_before_merge / time_merged << " elem/sec.)"
		<< std::endl;

		double time_total = time_aggregated + time_merged;
		std::cerr
		<< "Total in " << time_total
		<< " (" << n / time_total << " elem/sec.)"
		<< std::endl;
		std::cerr << "Size: " << maps[0].size() << std::endl << std::endl;
	}

	if (!method || method == 11)
	{
		/** Вариант 11.
		  * То же, что вариант 1, но при мердже, изменён порядок циклов,
		  *  что потенциально может дать лучшую кэш-локальность.
		  *
		  * На практике, разницы нет.
		  */

		std::vector<Map> maps(num_threads);

		Stopwatch watch;

		for (size_t i = 0; i < num_threads; ++i)
			pool.schedule(std::bind(aggregate1,
				std::ref(maps[i]),
				data.begin() + (data.size() * i) / num_threads,
				data.begin() + (data.size() * (i + 1)) / num_threads));

		pool.wait();

		watch.stop();
		double time_aggregated = watch.elapsedSeconds();
		std::cerr
		<< "Aggregated in " << time_aggregated
		<< " (" << n / time_aggregated << " elem/sec.)"
		<< std::endl;

		size_t size_before_merge = 0;
		std::cerr << "Sizes: ";
		for (size_t i = 0; i < num_threads; ++i)
		{
			std::cerr << (i == 0 ? "" : ", ") << maps[i].size();
			size_before_merge += maps[i].size();
		}
		std::cerr << std::endl;

		watch.restart();

		std::vector<Map::iterator> iterators(num_threads);
		for (size_t i = 1; i < num_threads; ++i)
			iterators[i] = maps[i].begin();

		while (true)
		{
			bool finish = true;
			for (size_t i = 1; i < num_threads; ++i)
			{
				if (iterators[i] == maps[i].end())
					continue;

				finish = false;
				maps[0][iterators[i]->first] += iterators[i]->second;
				++iterators[i];
			}

			if (finish)
				break;
		}

		watch.stop();
		double time_merged = watch.elapsedSeconds();
		std::cerr
		<< "Merged in " << time_merged
		<< " (" << size_before_merge / time_merged << " elem/sec.)"
		<< std::endl;

		double time_total = time_aggregated + time_merged;
		std::cerr
		<< "Total in " << time_total
		<< " (" << n / time_total << " elem/sec.)"
		<< std::endl;
		std::cerr << "Size: " << maps[0].size() << std::endl << std::endl;
	}

	if (!method || method == 2)
	{
		/** Вариант 2.
		  * В разных потоках агрегируем независимо в разные two-level хэш-таблицы.
		  * Затем сливаем их вместе, распараллелив по bucket-ам первого уровня.
		  * При использовании хэш-таблиц больших размеров (10 млн. элементов и больше),
		  *  и большого количества потоков (8-32), слияние является узким местом,
		  *  и преимущество в производительности достигает 4 раз.
		  */

		std::vector<MapTwoLevel> maps(num_threads);

		Stopwatch watch;

		for (size_t i = 0; i < num_threads; ++i)
			pool.schedule(std::bind(aggregate2,
				std::ref(maps[i]),
				data.begin() + (data.size() * i) / num_threads,
				data.begin() + (data.size() * (i + 1)) / num_threads));

		pool.wait();

		watch.stop();
		double time_aggregated = watch.elapsedSeconds();
		std::cerr
			<< "Aggregated in " << time_aggregated
			<< " (" << n / time_aggregated << " elem/sec.)"
			<< std::endl;

		size_t size_before_merge = 0;
		std::cerr << "Sizes: ";
		for (size_t i = 0; i < num_threads; ++i)
		{
			std::cerr << (i == 0 ? "" : ", ") << maps[i].size();
			size_before_merge += maps[i].size();
		}
		std::cerr << std::endl;

		watch.restart();

		for (size_t i = 0; i < MapTwoLevel::NUM_BUCKETS; ++i)
			pool.schedule(std::bind(merge2,
				&maps[0], num_threads, i));

		pool.wait();

		watch.stop();
		double time_merged = watch.elapsedSeconds();
		std::cerr
			<< "Merged in " << time_merged
			<< " (" << size_before_merge / time_merged << " elem/sec.)"
			<< std::endl;

		double time_total = time_aggregated + time_merged;
		std::cerr
			<< "Total in " << time_total
			<< " (" << n / time_total << " elem/sec.)"
			<< std::endl;

		std::cerr << "Size: " << maps[0].size() << std::endl << std::endl;
	}

	if (!method || method == 22)
	{
		std::vector<MapTwoLevel> maps(num_threads);

		Stopwatch watch;

		for (size_t i = 0; i < num_threads; ++i)
			pool.schedule(std::bind(aggregate22,
									std::ref(maps[i]),
									data.begin() + (data.size() * i) / num_threads,
									data.begin() + (data.size() * (i + 1)) / num_threads));

		pool.wait();

		watch.stop();
		double time_aggregated = watch.elapsedSeconds();
		std::cerr
		<< "Aggregated in " << time_aggregated
		<< " (" << n / time_aggregated << " elem/sec.)"
		<< std::endl;

		size_t size_before_merge = 0;
		std::cerr << "Sizes: ";
		for (size_t i = 0; i < num_threads; ++i)
		{
			std::cerr << (i == 0 ? "" : ", ") << maps[i].size();
			size_before_merge += maps[i].size();
		}
		std::cerr << std::endl;

		watch.restart();

		for (size_t i = 0; i < MapTwoLevel::NUM_BUCKETS; ++i)
			pool.schedule(std::bind(merge2,
									&maps[0], num_threads, i));

		pool.wait();

		watch.stop();
		double time_merged = watch.elapsedSeconds();
		std::cerr
		<< "Merged in " << time_merged
		<< " (" << size_before_merge / time_merged << " elem/sec.)"
		<< std::endl;

		double time_total = time_aggregated + time_merged;
		std::cerr
		<< "Total in " << time_total
		<< " (" << n / time_total << " elem/sec.)"
		<< std::endl;

		std::cerr << "Size: " << maps[0].size() << std::endl << std::endl;
	}

	if (!method || method == 3)
	{
		/** Вариант 3.
		  * В разных потоках агрегируем независимо в разные хэш-таблицы,
		  *  пока их размер не станет достаточно большим.
		  * Если размер локальной хэш-таблицы большой, и в ней нет элемента,
		  *  то вставляем его в одну глобальную хэш-таблицу, защищённую mutex-ом,
		  *  а если mutex не удалось захватить, то вставляем в локальную.
		  * Затем сливаем все локальные хэш-таблицы в глобальную.
		  * Этот метод плохой - много contention-а.
		  */

		std::vector<Map> local_maps(num_threads);
		Map global_map;
		Mutex mutex;

		Stopwatch watch;

		for (size_t i = 0; i < num_threads; ++i)
			pool.schedule(std::bind(aggregate3,
				std::ref(local_maps[i]),
				std::ref(global_map),
				std::ref(mutex),
				data.begin() + (data.size() * i) / num_threads,
				data.begin() + (data.size() * (i + 1)) / num_threads));

		pool.wait();

		watch.stop();
		double time_aggregated = watch.elapsedSeconds();
		std::cerr
			<< "Aggregated in " << time_aggregated
			<< " (" << n / time_aggregated << " elem/sec.)"
			<< std::endl;

		size_t size_before_merge = 0;
		std::cerr << "Sizes (local): ";
		for (size_t i = 0; i < num_threads; ++i)
		{
			std::cerr << (i == 0 ? "" : ", ") << local_maps[i].size();
			size_before_merge += local_maps[i].size();
		}
		std::cerr << std::endl;
		std::cerr << "Size (global): " << global_map.size() << std::endl;
		size_before_merge += global_map.size();

		watch.restart();

		for (size_t i = 0; i < num_threads; ++i)
			for (auto it = local_maps[i].begin(); it != local_maps[i].end(); ++it)
				global_map[it->first] += it->second;

		pool.wait();

		watch.stop();
		double time_merged = watch.elapsedSeconds();
		std::cerr
			<< "Merged in " << time_merged
			<< " (" << size_before_merge / time_merged << " elem/sec.)"
			<< std::endl;

		double time_total = time_aggregated + time_merged;
		std::cerr
			<< "Total in " << time_total
			<< " (" << n / time_total << " elem/sec.)"
			<< std::endl;

		std::cerr << "Size: " << global_map.size() << std::endl << std::endl;
	}

	if (!method || method == 33)
	{
		/** Вариант 33.
		 * В разных потоках агрегируем независимо в разные хэш-таблицы,
		 *  пока их размер не станет достаточно большим.
		 * Затем сбрасываем данные в глобальную хэш-таблицу, защищённую mutex-ом, и продолжаем.
		 */

		std::vector<Map> local_maps(num_threads);
		Map global_map;
		Mutex mutex;

		Stopwatch watch;

		for (size_t i = 0; i < num_threads; ++i)
			pool.schedule(std::bind(aggregate33,
				std::ref(local_maps[i]),
				std::ref(global_map),
				std::ref(mutex),
				data.begin() + (data.size() * i) / num_threads,
				data.begin() + (data.size() * (i + 1)) / num_threads));

		pool.wait();

		watch.stop();
		double time_aggregated = watch.elapsedSeconds();
		std::cerr
		<< "Aggregated in " << time_aggregated
		<< " (" << n / time_aggregated << " elem/sec.)"
		<< std::endl;

		size_t size_before_merge = 0;
		std::cerr << "Sizes (local): ";
		for (size_t i = 0; i < num_threads; ++i)
		{
			std::cerr << (i == 0 ? "" : ", ") << local_maps[i].size();
			size_before_merge += local_maps[i].size();
		}
		std::cerr << std::endl;
		std::cerr << "Size (global): " << global_map.size() << std::endl;
		size_before_merge += global_map.size();

		watch.restart();

		for (size_t i = 0; i < num_threads; ++i)
			for (auto it = local_maps[i].begin(); it != local_maps[i].end(); ++it)
				global_map[it->first] += it->second;

		pool.wait();

		watch.stop();
		double time_merged = watch.elapsedSeconds();
		std::cerr
		<< "Merged in " << time_merged
		<< " (" << size_before_merge / time_merged << " elem/sec.)"
		<< std::endl;

		double time_total = time_aggregated + time_merged;
		std::cerr
		<< "Total in " << time_total
		<< " (" << n / time_total << " elem/sec.)"
		<< std::endl;

		std::cerr << "Size: " << global_map.size() << std::endl << std::endl;
	}

	if (!method || method == 4)
	{
		/** Вариант 4.
		  * В разных потоках агрегируем независимо в разные хэш-таблицы,
		  *  пока их размер не станет достаточно большим.
		  * Если размер локальной хэш-таблицы большой, и в ней нет элемента,
		  *  то вставляем его в одну из 256 глобальных хэш-таблиц, каждая из которых под своим mutex-ом.
		  * Затем сливаем все локальные хэш-таблицы в глобальную.
		  * Этот метод не такой уж плохой при большом количестве потоков, но хуже второго.
		  */

		std::vector<Map> local_maps(num_threads);
		MapTwoLevel global_map;
		std::vector<Mutex> mutexes(MapTwoLevel::NUM_BUCKETS);

		Stopwatch watch;

		for (size_t i = 0; i < num_threads; ++i)
			pool.schedule(std::bind(aggregate4,
				std::ref(local_maps[i]),
				std::ref(global_map),
				&mutexes[0],
				data.begin() + (data.size() * i) / num_threads,
				data.begin() + (data.size() * (i + 1)) / num_threads));

		pool.wait();

		watch.stop();
		double time_aggregated = watch.elapsedSeconds();
		std::cerr
			<< "Aggregated in " << time_aggregated
			<< " (" << n / time_aggregated << " elem/sec.)"
			<< std::endl;

		size_t size_before_merge = 0;
		std::cerr << "Sizes (local): ";
		for (size_t i = 0; i < num_threads; ++i)
		{
			std::cerr << (i == 0 ? "" : ", ") << local_maps[i].size();
			size_before_merge += local_maps[i].size();
		}
		std::cerr << std::endl;

		size_t sum_size = global_map.size();
		std::cerr << "Size (global): " << sum_size << std::endl;
		size_before_merge += sum_size;

		watch.restart();

		for (size_t i = 0; i < num_threads; ++i)
			for (auto it = local_maps[i].begin(); it != local_maps[i].end(); ++it)
				global_map[it->first] += it->second;

		pool.wait();

		watch.stop();
		double time_merged = watch.elapsedSeconds();
		std::cerr
			<< "Merged in " << time_merged
			<< " (" << size_before_merge / time_merged << " elem/sec.)"
			<< std::endl;

		double time_total = time_aggregated + time_merged;
		std::cerr
			<< "Total in " << time_total
			<< " (" << n / time_total << " elem/sec.)"
			<< std::endl;

		std::cerr << "Size: " << global_map.size() << std::endl << std::endl;
	}

/*	if (!method || method == 5)
	{
	*/	/** Вариант 5.
		  * В разных потоках агрегируем независимо в разные хэш-таблицы,
		  *  пока их размер не станет достаточно большим.
		  * Если размер локальной хэш-таблицы большой, и в ней нет элемента,
		  *  то вставляем его в одну глобальную хэш-таблицу, содержащую маленькие защёлки в каждой ячейке,
		  *  а если защёлку не удалось захватить, то вставляем в локальную.
		  * Затем сливаем все локальные хэш-таблицы в глобальную.
		  */
/*
		Map local_maps[num_threads];
		MapSmallLocks global_map;

		Stopwatch watch;

		for (size_t i = 0; i < num_threads; ++i)
			pool.schedule(std::bind(aggregate5,
				std::ref(local_maps[i]),
				std::ref(global_map),
				data.begin() + (data.size() * i) / num_threads,
				data.begin() + (data.size() * (i + 1)) / num_threads));

		pool.wait();

		watch.stop();
		double time_aggregated = watch.elapsedSeconds();
		std::cerr
			<< "Aggregated in " << time_aggregated
			<< " (" << n / time_aggregated << " elem/sec.)"
			<< std::endl;

		size_t size_before_merge = 0;
		std::cerr << "Sizes (local): ";
		for (size_t i = 0; i < num_threads; ++i)
		{
			std::cerr << (i == 0 ? "" : ", ") << local_maps[i].size();
			size_before_merge += local_maps[i].size();
		}
		std::cerr << std::endl;
		std::cerr << "Size (global): " << global_map.size() << std::endl;
		size_before_merge += global_map.size();

		watch.restart();

		for (size_t i = 0; i < num_threads; ++i)
			for (auto it = local_maps[i].begin(); it != local_maps[i].end(); ++it)
				global_map.insert(std::make_pair(it->first, 0)).first->second += it->second;

		pool.wait();

		watch.stop();
		double time_merged = watch.elapsedSeconds();
		std::cerr
			<< "Merged in " << time_merged
			<< " (" << size_before_merge / time_merged << " elem/sec.)"
			<< std::endl;

		double time_total = time_aggregated + time_merged;
		std::cerr
			<< "Total in " << time_total
			<< " (" << n / time_total << " elem/sec.)"
			<< std::endl;

		std::cerr << "Size: " << global_map.size() << std::endl << std::endl;
	}*/

	/*if (!method || method == 6)
	{
		*//** Вариант 6.
		  * В разных потоках агрегируем независимо в разные хэш-таблицы.
		  * Затем "сливаем" их, проходя по ним в одинаковом порядке ключей.
		  * Довольно тормозной вариант.
		  */
/*
		std::vector<Map> maps(num_threads);

		Stopwatch watch;

		for (size_t i = 0; i < num_threads; ++i)
			pool.schedule(std::bind(aggregate1,
				std::ref(maps[i]),
				data.begin() + (data.size() * i) / num_threads,
				data.begin() + (data.size() * (i + 1)) / num_threads));

		pool.wait();

		watch.stop();
		double time_aggregated = watch.elapsedSeconds();
		std::cerr
			<< "Aggregated in " << time_aggregated
			<< " (" << n / time_aggregated << " elem/sec.)"
			<< std::endl;

		size_t size_before_merge = 0;
		std::cerr << "Sizes: ";
		for (size_t i = 0; i < num_threads; ++i)
		{
			std::cerr << (i == 0 ? "" : ", ") << maps[i].size();
			size_before_merge += maps[i].size();
		}
		std::cerr << std::endl;

		watch.restart();

		using Maps = std::vector<Map *>;
		Maps maps_to_merge(num_threads);
		for (size_t i = 0; i < num_threads; ++i)
			maps_to_merge[i] = &maps[i];

		size_t size = 0;

		for (size_t i = 0; i < 100; ++i)
		processMergedHashTables(maps_to_merge,
			[] (Map::value_type & dst, const Map::value_type & src) { dst.second += src.second; },
			[&] (const Map::value_type & dst) { ++size; });

		watch.stop();
		double time_merged = watch.elapsedSeconds();
		std::cerr
			<< "Merged in " << time_merged
			<< " (" << size_before_merge / time_merged << " elem/sec.)"
			<< std::endl;

		double time_total = time_aggregated + time_merged;
		std::cerr
			<< "Total in " << time_total
			<< " (" << n / time_total << " elem/sec.)"
			<< std::endl;
		std::cerr << "Size: " << size << std::endl << std::endl;
	}*/

	return 0;
}
void test(const Cont &)
{
  // Testing if all types are provided.

  typename Cont::value_type              t0;
  typename Cont::reference               t1 = t0;      CGAL_USE(t1);
  typename Cont::const_reference         t2 = t0;      CGAL_USE(t2);
  typename Cont::pointer                 t3 = &t0;
  typename Cont::const_pointer           t4 = &t0;     CGAL_USE(t4);
  typename Cont::size_type               t5 = 0;       CGAL_USE(t5);
  typename Cont::difference_type         t6 = t3-t3;   CGAL_USE(t6);
  typename Cont::iterator                t7;           CGAL_USE(t7);
  typename Cont::const_iterator          t8;           CGAL_USE(t8);
  typename Cont::reverse_iterator        t9;           CGAL_USE(t9);
  typename Cont::const_reverse_iterator  t10;          CGAL_USE(t10);
  typename Cont::allocator_type          t15;

  std::cout << "Testing empty containers." << std::endl;

  Cont c0, c1;
  Cont c2(t15);
  Cont c3(c2);
  Cont c4;
  c4 = c2;

  typedef std::vector<typename Cont::value_type> Vect;
  Vect v0;
  const Cont c5(v0.begin(), v0.end());
  Cont c6(c5.begin(), c5.end());
  typename Cont::allocator_type Al;
  Cont c7(c0.begin(), c0.end(), Al);
  Cont c8;
  c8.insert(c0.rbegin(), c0.rend());

  // test conversion iterator-> const_iterator.
  typename Cont::const_iterator t16 = c5.begin();  CGAL_USE(t16);
  assert(t16 == c5.begin());

  assert(c0 == c1);
  assert(! (c0 < c1));

  assert(check_empty(c0));
  assert(check_empty(c1));
  assert(check_empty(c2));
  assert(check_empty(c3));
  assert(check_empty(c4));
  assert(check_empty(c5));
  assert(check_empty(c6));
  assert(check_empty(c7));
  assert(check_empty(c8));

  c1.swap(c0);

  assert(check_empty(c0));
  assert(check_empty(c1));

  c1.merge(c0);

  assert(check_empty(c0));
  assert(check_empty(c1));

  typename Cont::allocator_type  t20 = c0.get_allocator();

  std::cout << "Now filling some containers" << std::endl;

  Vect v1(10000);
  Cont c9(v1.begin(), v1.end());

  assert(c9.size() == v1.size());
  assert(c9.max_size() >= v1.size());
  assert(c9.capacity() >= c9.size());

  Cont c10 = c9;

  assert(c10 == c9);
  assert(c10.size() == v1.size());
  assert(c10.max_size() >= v1.size());
  assert(c10.capacity() >= c10.size());

  c9.clear();

  assert(check_empty(c9));
  assert(c9.capacity() >= c9.size());
  assert(c0 == c9);

  c9.merge(c10);
  c10.swap(c9);

  assert(check_empty(c9));
  assert(c9.capacity() >= c9.size());

  assert(c10.size() == v1.size());
  assert(c10.max_size() >= v1.size());
  assert(c10.capacity() >= c10.size());

  std::cout << "Testing insertion methods" << std::endl;

  c9.assign(c10.begin(), c10.end());

  assert(c9 == c10);

  c10.assign(c9.begin(), c9.end());

  assert(c9 == c10);

  c9.insert(c10.begin(), c10.end());

  assert(c9.size() == 2*v1.size());

  c9.clear();

  assert(c9 != c10);

  c9.insert(c10.begin(), c10.end());

  assert(c9.size() == v1.size());
  assert(c9 == c10);


  typename Cont::iterator it = c9.iterator_to(*c9.begin());
  assert(it == c9.begin());
  typename Cont::const_iterator cit = c9.iterator_to(const_cast<typename Cont::const_reference>(*c9.begin()));
  assert(cit == c9.begin());

  typename Cont::iterator s_it = Cont::s_iterator_to(*c9.begin());
  assert(s_it == c9.begin());
  typename Cont::const_iterator s_cit = Cont::s_iterator_to(const_cast<typename Cont::const_reference>(*c9.begin()));
  assert(s_cit == c9.begin());


  c10 = Cont();

  assert(check_empty(c10));

  for(typename Vect::const_iterator it = v1.begin(); it != v1.end(); ++it)
    c10.insert(*it);

  assert(c10.size() == v1.size());
  assert(c9 == c10);

  c9.erase(c9.begin());
  c9.erase(c9.begin());

  assert(c9.size() == v1.size() - 2);

  // test reserve
  /*Cont c11;
  c11.reserve(v1.size());
  for(typename Vect::const_iterator it = v1.begin(); it != v1.end(); ++it)
    c11.insert(*it);
  
  assert(c11.size() == v1.size());
  assert(c10 == c11);*/

  // owns() and owns_dereferencable().
  for(typename Cont::const_iterator it = c9.begin(), end = c9.end(); it != end; ++it) {
    assert(c9.owns(it));
    assert(c9.owns_dereferencable(it));
    assert(! c10.owns(it));
    assert(! c10.owns_dereferencable(it));
  }
  assert(c9.owns(c9.end()));
  assert(! c9.owns_dereferencable(c9.end()));


  c9.erase(c9.begin(), c9.end());

  assert(check_empty(c9));
  
  std::cout << "Testing parallel insertion" << std::endl;
  {
  Cont c11;
  Vect v11(1000000);
  std::vector<typename Cont::iterator> iterators(v11.size());
  tbb::parallel_for(
    tbb::blocked_range<size_t>( 0, v11.size() ),
    Insert_in_CCC_functor<Vect, Cont>(v11, c11, iterators)
  );
  assert(c11.size() == v11.size());
  
  std::cout << "Testing parallel erasure" << std::endl;
  tbb::parallel_for(
    tbb::blocked_range<size_t>( 0, v11.size() ),
    Erase_in_CCC_functor<Cont>(c11, iterators)
  );
  assert(c11.empty());
  }

  std::cout << "Testing parallel insertion AND erasure" << std::endl;
  {
  Cont c12;
  Vect v12(1000000);
  std::vector<tbb::atomic<bool> > free_elements(v12.size());
  for(typename std::vector<tbb::atomic<bool> >::iterator 
    it = free_elements.begin(), end = free_elements.end(); it != end; ++it) 
  {
    *it = true;
  }
    
  tbb::atomic<unsigned int> num_erasures; 
  num_erasures = 0;
  std::vector<typename Cont::iterator> iterators(v12.size());
  tbb::parallel_for(
    tbb::blocked_range<size_t>( 0, v12.size() ),
    Insert_and_erase_in_CCC_functor<Vect, Cont>(
      v12, c12, iterators, free_elements, num_erasures)
  );
  assert(c12.size() == v12.size() - num_erasures);
  }
}
Beispiel #11
0
int
run_program(int argc, char **argv)
{
  CommandLineArgument<int> number_of_list_files;
  int next_command_line_argument = 0;
  bool echo_commands = false;

  for (int i = 1; ((i < argc) && (!have_argument_p(number_of_list_files))); i++) {
    std::string argument(argv[i]);
    if (argument == "--help") {
      print_usage();
      return 0;
    } else if (argument == "--echo") {
      echo_commands = true;
    } else if (!have_argument_p(number_of_list_files)) {
      number_of_list_files = argument;
      next_command_line_argument = i + 1;
    }
  }

  if (!have_argument_p(number_of_list_files)) {
    print_usage();
    return -1;
  }
  
  if (*number_of_list_files == 0)
    return 0;

  const int minimum_number_of_arguments = 0
    + *number_of_list_files
    + 1 // the command;
    + 0;

  if ((argc - next_command_line_argument) < minimum_number_of_arguments)
    throw make_runtime_error("Not enough arguments available for processing.\n");

  std::list<std::string> standard_input_list;
  bool have_read_standard_input_list = false;

  std::vector<std::string> list_filenames(*number_of_list_files);
  std::vector<std::list<std::string> > lists(*number_of_list_files);  
  for (int i = 0; i < *number_of_list_files; i++) {    
    int argument_index = i + next_command_line_argument;
    std::string list_file = argv[argument_index];
    std::list<std::string> l;
    
    if ((list_file == "-") && (have_read_standard_input_list)) {
      l = standard_input_list;
    } else if ((list_file == "-")) {
      standard_input_list = read_list(std::cin);
      have_read_standard_input_list = true;
      l = standard_input_list;
    } else if (!file_exists_p(list_file)) {
      throw make_runtime_error("List file %s does not exist.", list_file.c_str());
    } else {
      l = read_list(list_file.c_str());
    }

    list_filenames[i] = list_file;
    lists[i] = l;
  }
  next_command_line_argument += *number_of_list_files;

  // read the command and its options.
  std::string command(argv[next_command_line_argument]);
  std::list<std::string> command_arguments;
  std::copy(argv + next_command_line_argument + 1, argv + argc, std::back_inserter(command_arguments));

  // check all lists are the same size.
  for (size_t i = 0; i < lists.size(); i++) {
    if (lists[i].size() != lists[0].size())
      throw make_runtime_error("The number of entires in list %s (%d) differs to %s (%d).", 
			       list_filenames[i].c_str(),
			       lists[i].size(),
			       list_filenames[0].c_str(),
			       lists[0].size());
  }

  EchoRunner echo_runner(std::cout);
  ForkRunner fork_runner;

  Runner *runner = 0;
  if (echo_commands) 
    runner = &echo_runner;  
  else
    runner = &fork_runner;

  assert(runner);
  std::vector<std::list<std::string>::const_iterator> iterators(*number_of_list_files);
  for (int i = 0; i < *number_of_list_files; i++)
    iterators[i] = lists[i].begin();

  for (size_t i = 0; i < lists[0].size(); i++) {
    std::string invocation_command = command;
    std::list<std::string> invocation_arguments;
    std::copy(command_arguments.begin(), command_arguments.end(), std::back_inserter(invocation_arguments));
    for (size_t j = 0; j < iterators.size(); j++) {
      invocation_arguments.push_back(*iterators[j]);
      iterators[j]++;
    }

    runner->perform(invocation_command, invocation_arguments);
  }

  return 0;
}