const indri::infnet::InferenceNetwork::MAllResults& indri::infnet::InferenceNetwork::evaluate() {
// count this query occurrence
_repository.countQuery();
// fetch the current index state
indri::collection::Repository::index_state indexes = _repository.indexes();
for( size_t i=0; i<indexes->size(); i++ ) {
indri::index::Index& index = *(*indexes)[i];
indri::thread::ScopedLock iterators( index.iteratorLock() );
indri::thread::ScopedLock statistics( index.statisticsLock() );
_indexChanged( index );
statistics.unlock();
// evaluate query against the index
_evaluateIndex( index );
// remove all the iterators
_indexFinished( index );
}
_results.clear();
for( size_t i=0; i<_evaluators.size(); i++ ) {
_results[ _evaluators[i]->getName() ] = _evaluators[i]->getResults();
}
return _results;
}
static void NO_INLINE execute(Map ** source_maps, size_t num_maps, Map *& result_map,
Merger && merger,
ThreadPool &)
{
std::vector<typename Map::iterator> iterators(num_maps);
for (size_t i = 1; i < num_maps; ++i)
iterators[i] = source_maps[i]->begin();
result_map = source_maps[0];
while (true)
{
bool finish = true;
for (size_t i = 1; i < num_maps; ++i)
{
if (iterators[i] == source_maps[i]->end())
continue;
finish = false;
merger((*result_map)[iterators[i]->first], iterators[i]->second);
++iterators[i];
}
if (finish)
break;
}
}
void ObjectsToCostmap::setCostInPolygon(const grid_map::Polygon& polygon, const std::string& gridmap_layer_name,
const float score, grid_map::GridMap& objects_costmap)
{
grid_map::PolygonIterator iterators(objects_costmap, polygon);
for (grid_map::PolygonIterator iterator(objects_costmap, polygon); !iterator.isPastEnd(); ++iterator)
{
const float current_score = objects_costmap.at(gridmap_layer_name, *iterator);
if (score > current_score)
{
objects_costmap.at(gridmap_layer_name, *iterator) = score;
}
}
}
PhaseDetector::PhaseDetector(std::string input_filename, unsigned int max_phase_sz, unsigned int elems_to_read) :
max_phase_size(max_phase_sz),
max_elems(elems_to_read),
has_new_phase(false)
{
std::ifstream input_file(input_filename);
std::vector<int64_t> line_values;
std::string input_str;
if(input_file)
{
///reading the number of entries and the number of iterators
std::getline(input_file, input_str, ' ');
entries_num = std::stol(input_str);
std::getline(input_file, input_str);
iterator_num = std::stol(input_str);
///reading the iterators' bounds
for(unsigned int i = 0; i < iterator_num - 1; i++)
{
std::getline(input_file, input_str, ' ');
bounds.insert(bounds.begin(),std::stol(input_str));
}
std::getline(input_file, input_str);
bounds.insert(bounds.begin(), std::stol(input_str));
///reading the memory positions and their corresponding iteration vectors
for(unsigned int i = 0; i < max_elems; i++)
{
std::getline(input_file, input_str,'\n');
if(input_file.eof()) break;
line_values = splitEntry(input_str);
std::vector<int64_t> iterators(line_values.begin() + 1, line_values.end());
addr_vector.push_back(line_values[0]);
iterators_vector.push_back(iterators);
}
next_start = 0;
}
else
{
cerr << "Couldn't open the input file\n";
exit(EXIT_FAILURE);
}
input_file.close();
}
int main() {
initWithSingleElement();
initWithContainers();
initWithIterators();
copyAndMove();
appendAndJoin();
iterators();
modify();
operations();
compare();
assignment();
emplace();
merge();
sort();
resize();
}
virtual bool run(OperationContext* txn, const string& dbname, BSONObj& cmdObj, int options,
string& errmsg, BSONObjBuilder& result,
bool fromRepl = false ) {
NamespaceString ns( dbname, cmdObj[name].String() );
AutoGetCollectionForRead ctx(txn, ns.ns());
Collection* collection = ctx.getCollection();
if ( !collection )
return appendCommandStatus( result,
Status( ErrorCodes::NamespaceNotFound,
str::stream() <<
"ns does not exist: " << ns.ns() ) );
size_t numCursors = static_cast<size_t>( cmdObj["numCursors"].numberInt() );
if ( numCursors == 0 || numCursors > 10000 )
return appendCommandStatus( result,
Status( ErrorCodes::BadValue,
str::stream() <<
"numCursors has to be between 1 and 10000" <<
" was: " << numCursors ) );
OwnedPointerVector<RecordIterator> iterators(collection->getManyIterators(txn));
if (iterators.size() < numCursors) {
numCursors = iterators.size();
}
OwnedPointerVector<PlanExecutor> execs;
for ( size_t i = 0; i < numCursors; i++ ) {
WorkingSet* ws = new WorkingSet();
MultiIteratorStage* mis = new MultiIteratorStage(txn, ws, collection);
// Takes ownership of 'ws' and 'mis'.
auto_ptr<PlanExecutor> curExec(new PlanExecutor(txn, ws, mis, collection));
// Each of the plan executors should yield automatically. We pass "false" to
// indicate that 'curExec' should not register itself, as it will get registered
// by ClientCursor instead.
curExec->setYieldPolicy(PlanExecutor::YIELD_AUTO, false);
// Need to save state while yielding locks between now and newGetMore.
curExec->saveState();
execs.push_back(curExec.release());
}
// transfer iterators to executors using a round-robin distribution.
// TODO consider using a common work queue once invalidation issues go away.
for (size_t i = 0; i < iterators.size(); i++) {
PlanExecutor* theExec = execs[i % execs.size()];
MultiIteratorStage* mis = static_cast<MultiIteratorStage*>(theExec->getRootStage());
mis->addIterator(iterators.releaseAt(i));
}
{
BSONArrayBuilder bucketsBuilder;
for (size_t i = 0; i < execs.size(); i++) {
// transfer ownership of an executor to the ClientCursor (which manages its own
// lifetime).
ClientCursor* cc = new ClientCursor( collection, execs.releaseAt(i) );
// we are mimicking the aggregation cursor output here
// that is why there are ns, ok and empty firstBatch
BSONObjBuilder threadResult;
{
BSONObjBuilder cursor;
cursor.appendArray( "firstBatch", BSONObj() );
cursor.append( "ns", ns );
cursor.append( "id", cc->cursorid() );
threadResult.append( "cursor", cursor.obj() );
}
threadResult.appendBool( "ok", 1 );
bucketsBuilder.append( threadResult.obj() );
}
result.appendArray( "cursors", bucketsBuilder.obj() );
}
return true;
}
/*
* Merge the runs from the input file into the output file.
*/
bool MergeSorter::merge_pass()
{
// Check how many runs we're merging.
const unsigned int RECORDS_PER_PAGE = records_per_page(_page_size);
int run_count = (_records_pages + (_run_length - 1)) / _run_length;
// Merge runs while there are runs left.
int run_index;
for (run_index = 0; run_index < run_count; run_index += _k) {
// Check how many we're merging now.
int merge_count = run_count - run_index;
if (merge_count > _k) {
merge_count = _k;
}
std::vector<RunIterator> iterators(merge_count);
// Set up the iterators.
int i;
for (i = 0; i < merge_count; ++i) {
int current_index = run_index + i;
int run_page_index = current_index * _run_length;
int run_offset = current_index * _run_length * _page_size;
int page_buffer_offset = ((i + 1) * _page_size);
RunIterator& iter = iterators[i];
iter.set_buffer(_pages + page_buffer_offset, _page_size);
// Calculate how many pages this run has.
int length = _records_pages - run_page_index;
if (length > _run_length) {
length = _run_length;
}
iter.set_run(_run_input, run_offset, length);
// Move to first value.
if (!iter.next()) {
return false;
}
}
// We'll remove iterators when we're done with them, so loop until we have no more.
// Use the first page as the output buffer.
Record *output_buffer = reinterpret_cast<Record*>(_pages);
unsigned int output_record_index = 0;
while (!iterators.empty()) {
// Find the minimum current value.
Record *smallest_record = NULL;
std::vector<RunIterator>::iterator smallest_j;
std::vector<RunIterator>::iterator j;
for (j = iterators.begin(); j != iterators.end(); ++j) {
RunIterator& iter = *j;
// Compare this record to current lowest.
Record *record = iter.current();
if ((smallest_record == NULL) || (record_compare(record, smallest_record) < 0)) {
smallest_record = record;
smallest_j = j;
}
}
// Write the smallest record to merge page.
memcpy(&output_buffer[output_record_index], smallest_record, RECORD_SIZE);
++output_record_index;
// Push smallest iterator to next value.
RunIterator& smallest_run = *smallest_j;
if (!smallest_run.next()) {
return false;
}
// Remove this iterator if it's empty.
if (smallest_run.current() == NULL) {
iterators.erase(smallest_j);
}
bool is_done = iterators.empty();
bool is_full = output_record_index == RECORDS_PER_PAGE;
// Null out the end entry if page isn't full.
if (is_done && !is_full) {
Record *last = &output_buffer[output_record_index];
(*last)[RECORD_SIZE - 1] = '\0';
}
// Write the last page if full or last record.
else if (is_full || is_done) {
size_t written = fwrite(output_buffer, _page_size, 1, _run_output);
if (written != 1) {
fprintf(stderr, "Failed to write output buffer page to disk.\n");
return false;
}
output_record_index = 0;
}
}
}
// We merged merge_maximum at a time, so runs are that many times longer.
_run_length *= _k;
printf("Runs are length %d.\n", _run_length);
return true;
}
virtual bool run( const string& dbname, BSONObj& cmdObj, int options,
string& errmsg, BSONObjBuilder& result,
bool fromRepl = false ) {
NamespaceString ns( dbname, cmdObj[name].String() );
Client::ReadContext ctx(ns.ns());
Database* db = ctx.ctx().db();
Collection* collection = db->getCollection( ns );
if ( !collection )
return appendCommandStatus( result,
Status( ErrorCodes::NamespaceNotFound,
str::stream() <<
"ns does not exist: " << ns.ns() ) );
size_t numCursors = static_cast<size_t>( cmdObj["numCursors"].numberInt() );
if ( numCursors == 0 || numCursors > 10000 )
return appendCommandStatus( result,
Status( ErrorCodes::BadValue,
str::stream() <<
"numCursors has to be between 1 and 10000" <<
" was: " << numCursors ) );
OwnedPointerVector<RecordIterator> iterators(collection->getManyIterators());
if (iterators.size() < numCursors) {
numCursors = iterators.size();
}
OwnedPointerVector<MultiIteratorRunner> runners;
for ( size_t i = 0; i < numCursors; i++ ) {
runners.push_back(new MultiIteratorRunner(ns.ns(), collection));
}
// transfer iterators to runners using a round-robin distribution.
// TODO consider using a common work queue once invalidation issues go away.
for (size_t i = 0; i < iterators.size(); i++) {
runners[i % runners.size()]->addIterator(iterators.releaseAt(i));
}
{
BSONArrayBuilder bucketsBuilder;
for (size_t i = 0; i < runners.size(); i++) {
// transfer ownership of a runner to the ClientCursor (which manages its own
// lifetime).
ClientCursor* cc = new ClientCursor( collection, runners.releaseAt(i) );
// we are mimicking the aggregation cursor output here
// that is why there are ns, ok and empty firstBatch
BSONObjBuilder threadResult;
{
BSONObjBuilder cursor;
cursor.appendArray( "firstBatch", BSONObj() );
cursor.append( "ns", ns );
cursor.append( "id", cc->cursorid() );
threadResult.append( "cursor", cursor.obj() );
}
threadResult.appendBool( "ok", 1 );
bucketsBuilder.append( threadResult.obj() );
}
result.appendArray( "cursors", bucketsBuilder.obj() );
}
return true;
}
int main(int argc, char ** argv)
{
size_t n = atoi(argv[1]);
size_t num_threads = atoi(argv[2]);
size_t method = argc <= 3 ? 0 : atoi(argv[3]);
std::cerr << std::fixed << std::setprecision(2);
ThreadPool pool(num_threads);
Source data(n);
{
Stopwatch watch;
DB::ReadBufferFromFileDescriptor in1(STDIN_FILENO);
DB::CompressedReadBuffer in2(in1);
in2.readStrict(reinterpret_cast<char*>(&data[0]), sizeof(data[0]) * n);
watch.stop();
std::cerr << std::fixed << std::setprecision(2)
<< "Vector. Size: " << n
<< ", elapsed: " << watch.elapsedSeconds()
<< " (" << n / watch.elapsedSeconds() << " elem/sec.)"
<< std::endl << std::endl;
}
if (!method || method == 1)
{
/** Вариант 1.
* В разных потоках агрегируем независимо в разные хэш-таблицы.
* Затем сливаем их вместе.
*/
std::vector<Map> maps(num_threads);
Stopwatch watch;
for (size_t i = 0; i < num_threads; ++i)
pool.schedule(std::bind(aggregate1,
std::ref(maps[i]),
data.begin() + (data.size() * i) / num_threads,
data.begin() + (data.size() * (i + 1)) / num_threads));
pool.wait();
watch.stop();
double time_aggregated = watch.elapsedSeconds();
std::cerr
<< "Aggregated in " << time_aggregated
<< " (" << n / time_aggregated << " elem/sec.)"
<< std::endl;
size_t size_before_merge = 0;
std::cerr << "Sizes: ";
for (size_t i = 0; i < num_threads; ++i)
{
std::cerr << (i == 0 ? "" : ", ") << maps[i].size();
size_before_merge += maps[i].size();
}
std::cerr << std::endl;
watch.restart();
for (size_t i = 1; i < num_threads; ++i)
for (auto it = maps[i].begin(); it != maps[i].end(); ++it)
maps[0][it->first] += it->second;
watch.stop();
double time_merged = watch.elapsedSeconds();
std::cerr
<< "Merged in " << time_merged
<< " (" << size_before_merge / time_merged << " elem/sec.)"
<< std::endl;
double time_total = time_aggregated + time_merged;
std::cerr
<< "Total in " << time_total
<< " (" << n / time_total << " elem/sec.)"
<< std::endl;
std::cerr << "Size: " << maps[0].size() << std::endl << std::endl;
}
if (!method || method == 12)
{
/** То же самое, но с оптимизацией для подряд идущих одинаковых значений.
*/
std::vector<Map> maps(num_threads);
Stopwatch watch;
for (size_t i = 0; i < num_threads; ++i)
pool.schedule(std::bind(aggregate12,
std::ref(maps[i]),
data.begin() + (data.size() * i) / num_threads,
data.begin() + (data.size() * (i + 1)) / num_threads));
pool.wait();
watch.stop();
double time_aggregated = watch.elapsedSeconds();
std::cerr
<< "Aggregated in " << time_aggregated
<< " (" << n / time_aggregated << " elem/sec.)"
<< std::endl;
size_t size_before_merge = 0;
std::cerr << "Sizes: ";
for (size_t i = 0; i < num_threads; ++i)
{
std::cerr << (i == 0 ? "" : ", ") << maps[i].size();
size_before_merge += maps[i].size();
}
std::cerr << std::endl;
watch.restart();
for (size_t i = 1; i < num_threads; ++i)
for (auto it = maps[i].begin(); it != maps[i].end(); ++it)
maps[0][it->first] += it->second;
watch.stop();
double time_merged = watch.elapsedSeconds();
std::cerr
<< "Merged in " << time_merged
<< " (" << size_before_merge / time_merged << " elem/sec.)"
<< std::endl;
double time_total = time_aggregated + time_merged;
std::cerr
<< "Total in " << time_total
<< " (" << n / time_total << " elem/sec.)"
<< std::endl;
std::cerr << "Size: " << maps[0].size() << std::endl << std::endl;
}
if (!method || method == 11)
{
/** Вариант 11.
* То же, что вариант 1, но при мердже, изменён порядок циклов,
* что потенциально может дать лучшую кэш-локальность.
*
* На практике, разницы нет.
*/
std::vector<Map> maps(num_threads);
Stopwatch watch;
for (size_t i = 0; i < num_threads; ++i)
pool.schedule(std::bind(aggregate1,
std::ref(maps[i]),
data.begin() + (data.size() * i) / num_threads,
data.begin() + (data.size() * (i + 1)) / num_threads));
pool.wait();
watch.stop();
double time_aggregated = watch.elapsedSeconds();
std::cerr
<< "Aggregated in " << time_aggregated
<< " (" << n / time_aggregated << " elem/sec.)"
<< std::endl;
size_t size_before_merge = 0;
std::cerr << "Sizes: ";
for (size_t i = 0; i < num_threads; ++i)
{
std::cerr << (i == 0 ? "" : ", ") << maps[i].size();
size_before_merge += maps[i].size();
}
std::cerr << std::endl;
watch.restart();
std::vector<Map::iterator> iterators(num_threads);
for (size_t i = 1; i < num_threads; ++i)
iterators[i] = maps[i].begin();
while (true)
{
bool finish = true;
for (size_t i = 1; i < num_threads; ++i)
{
if (iterators[i] == maps[i].end())
continue;
finish = false;
maps[0][iterators[i]->first] += iterators[i]->second;
++iterators[i];
}
if (finish)
break;
}
watch.stop();
double time_merged = watch.elapsedSeconds();
std::cerr
<< "Merged in " << time_merged
<< " (" << size_before_merge / time_merged << " elem/sec.)"
<< std::endl;
double time_total = time_aggregated + time_merged;
std::cerr
<< "Total in " << time_total
<< " (" << n / time_total << " elem/sec.)"
<< std::endl;
std::cerr << "Size: " << maps[0].size() << std::endl << std::endl;
}
if (!method || method == 2)
{
/** Вариант 2.
* В разных потоках агрегируем независимо в разные two-level хэш-таблицы.
* Затем сливаем их вместе, распараллелив по bucket-ам первого уровня.
* При использовании хэш-таблиц больших размеров (10 млн. элементов и больше),
* и большого количества потоков (8-32), слияние является узким местом,
* и преимущество в производительности достигает 4 раз.
*/
std::vector<MapTwoLevel> maps(num_threads);
Stopwatch watch;
for (size_t i = 0; i < num_threads; ++i)
pool.schedule(std::bind(aggregate2,
std::ref(maps[i]),
data.begin() + (data.size() * i) / num_threads,
data.begin() + (data.size() * (i + 1)) / num_threads));
pool.wait();
watch.stop();
double time_aggregated = watch.elapsedSeconds();
std::cerr
<< "Aggregated in " << time_aggregated
<< " (" << n / time_aggregated << " elem/sec.)"
<< std::endl;
size_t size_before_merge = 0;
std::cerr << "Sizes: ";
for (size_t i = 0; i < num_threads; ++i)
{
std::cerr << (i == 0 ? "" : ", ") << maps[i].size();
size_before_merge += maps[i].size();
}
std::cerr << std::endl;
watch.restart();
for (size_t i = 0; i < MapTwoLevel::NUM_BUCKETS; ++i)
pool.schedule(std::bind(merge2,
&maps[0], num_threads, i));
pool.wait();
watch.stop();
double time_merged = watch.elapsedSeconds();
std::cerr
<< "Merged in " << time_merged
<< " (" << size_before_merge / time_merged << " elem/sec.)"
<< std::endl;
double time_total = time_aggregated + time_merged;
std::cerr
<< "Total in " << time_total
<< " (" << n / time_total << " elem/sec.)"
<< std::endl;
std::cerr << "Size: " << maps[0].size() << std::endl << std::endl;
}
if (!method || method == 22)
{
std::vector<MapTwoLevel> maps(num_threads);
Stopwatch watch;
for (size_t i = 0; i < num_threads; ++i)
pool.schedule(std::bind(aggregate22,
std::ref(maps[i]),
data.begin() + (data.size() * i) / num_threads,
data.begin() + (data.size() * (i + 1)) / num_threads));
pool.wait();
watch.stop();
double time_aggregated = watch.elapsedSeconds();
std::cerr
<< "Aggregated in " << time_aggregated
<< " (" << n / time_aggregated << " elem/sec.)"
<< std::endl;
size_t size_before_merge = 0;
std::cerr << "Sizes: ";
for (size_t i = 0; i < num_threads; ++i)
{
std::cerr << (i == 0 ? "" : ", ") << maps[i].size();
size_before_merge += maps[i].size();
}
std::cerr << std::endl;
watch.restart();
for (size_t i = 0; i < MapTwoLevel::NUM_BUCKETS; ++i)
pool.schedule(std::bind(merge2,
&maps[0], num_threads, i));
pool.wait();
watch.stop();
double time_merged = watch.elapsedSeconds();
std::cerr
<< "Merged in " << time_merged
<< " (" << size_before_merge / time_merged << " elem/sec.)"
<< std::endl;
double time_total = time_aggregated + time_merged;
std::cerr
<< "Total in " << time_total
<< " (" << n / time_total << " elem/sec.)"
<< std::endl;
std::cerr << "Size: " << maps[0].size() << std::endl << std::endl;
}
if (!method || method == 3)
{
/** Вариант 3.
* В разных потоках агрегируем независимо в разные хэш-таблицы,
* пока их размер не станет достаточно большим.
* Если размер локальной хэш-таблицы большой, и в ней нет элемента,
* то вставляем его в одну глобальную хэш-таблицу, защищённую mutex-ом,
* а если mutex не удалось захватить, то вставляем в локальную.
* Затем сливаем все локальные хэш-таблицы в глобальную.
* Этот метод плохой - много contention-а.
*/
std::vector<Map> local_maps(num_threads);
Map global_map;
Mutex mutex;
Stopwatch watch;
for (size_t i = 0; i < num_threads; ++i)
pool.schedule(std::bind(aggregate3,
std::ref(local_maps[i]),
std::ref(global_map),
std::ref(mutex),
data.begin() + (data.size() * i) / num_threads,
data.begin() + (data.size() * (i + 1)) / num_threads));
pool.wait();
watch.stop();
double time_aggregated = watch.elapsedSeconds();
std::cerr
<< "Aggregated in " << time_aggregated
<< " (" << n / time_aggregated << " elem/sec.)"
<< std::endl;
size_t size_before_merge = 0;
std::cerr << "Sizes (local): ";
for (size_t i = 0; i < num_threads; ++i)
{
std::cerr << (i == 0 ? "" : ", ") << local_maps[i].size();
size_before_merge += local_maps[i].size();
}
std::cerr << std::endl;
std::cerr << "Size (global): " << global_map.size() << std::endl;
size_before_merge += global_map.size();
watch.restart();
for (size_t i = 0; i < num_threads; ++i)
for (auto it = local_maps[i].begin(); it != local_maps[i].end(); ++it)
global_map[it->first] += it->second;
pool.wait();
watch.stop();
double time_merged = watch.elapsedSeconds();
std::cerr
<< "Merged in " << time_merged
<< " (" << size_before_merge / time_merged << " elem/sec.)"
<< std::endl;
double time_total = time_aggregated + time_merged;
std::cerr
<< "Total in " << time_total
<< " (" << n / time_total << " elem/sec.)"
<< std::endl;
std::cerr << "Size: " << global_map.size() << std::endl << std::endl;
}
if (!method || method == 33)
{
/** Вариант 33.
* В разных потоках агрегируем независимо в разные хэш-таблицы,
* пока их размер не станет достаточно большим.
* Затем сбрасываем данные в глобальную хэш-таблицу, защищённую mutex-ом, и продолжаем.
*/
std::vector<Map> local_maps(num_threads);
Map global_map;
Mutex mutex;
Stopwatch watch;
for (size_t i = 0; i < num_threads; ++i)
pool.schedule(std::bind(aggregate33,
std::ref(local_maps[i]),
std::ref(global_map),
std::ref(mutex),
data.begin() + (data.size() * i) / num_threads,
data.begin() + (data.size() * (i + 1)) / num_threads));
pool.wait();
watch.stop();
double time_aggregated = watch.elapsedSeconds();
std::cerr
<< "Aggregated in " << time_aggregated
<< " (" << n / time_aggregated << " elem/sec.)"
<< std::endl;
size_t size_before_merge = 0;
std::cerr << "Sizes (local): ";
for (size_t i = 0; i < num_threads; ++i)
{
std::cerr << (i == 0 ? "" : ", ") << local_maps[i].size();
size_before_merge += local_maps[i].size();
}
std::cerr << std::endl;
std::cerr << "Size (global): " << global_map.size() << std::endl;
size_before_merge += global_map.size();
watch.restart();
for (size_t i = 0; i < num_threads; ++i)
for (auto it = local_maps[i].begin(); it != local_maps[i].end(); ++it)
global_map[it->first] += it->second;
pool.wait();
watch.stop();
double time_merged = watch.elapsedSeconds();
std::cerr
<< "Merged in " << time_merged
<< " (" << size_before_merge / time_merged << " elem/sec.)"
<< std::endl;
double time_total = time_aggregated + time_merged;
std::cerr
<< "Total in " << time_total
<< " (" << n / time_total << " elem/sec.)"
<< std::endl;
std::cerr << "Size: " << global_map.size() << std::endl << std::endl;
}
if (!method || method == 4)
{
/** Вариант 4.
* В разных потоках агрегируем независимо в разные хэш-таблицы,
* пока их размер не станет достаточно большим.
* Если размер локальной хэш-таблицы большой, и в ней нет элемента,
* то вставляем его в одну из 256 глобальных хэш-таблиц, каждая из которых под своим mutex-ом.
* Затем сливаем все локальные хэш-таблицы в глобальную.
* Этот метод не такой уж плохой при большом количестве потоков, но хуже второго.
*/
std::vector<Map> local_maps(num_threads);
MapTwoLevel global_map;
std::vector<Mutex> mutexes(MapTwoLevel::NUM_BUCKETS);
Stopwatch watch;
for (size_t i = 0; i < num_threads; ++i)
pool.schedule(std::bind(aggregate4,
std::ref(local_maps[i]),
std::ref(global_map),
&mutexes[0],
data.begin() + (data.size() * i) / num_threads,
data.begin() + (data.size() * (i + 1)) / num_threads));
pool.wait();
watch.stop();
double time_aggregated = watch.elapsedSeconds();
std::cerr
<< "Aggregated in " << time_aggregated
<< " (" << n / time_aggregated << " elem/sec.)"
<< std::endl;
size_t size_before_merge = 0;
std::cerr << "Sizes (local): ";
for (size_t i = 0; i < num_threads; ++i)
{
std::cerr << (i == 0 ? "" : ", ") << local_maps[i].size();
size_before_merge += local_maps[i].size();
}
std::cerr << std::endl;
size_t sum_size = global_map.size();
std::cerr << "Size (global): " << sum_size << std::endl;
size_before_merge += sum_size;
watch.restart();
for (size_t i = 0; i < num_threads; ++i)
for (auto it = local_maps[i].begin(); it != local_maps[i].end(); ++it)
global_map[it->first] += it->second;
pool.wait();
watch.stop();
double time_merged = watch.elapsedSeconds();
std::cerr
<< "Merged in " << time_merged
<< " (" << size_before_merge / time_merged << " elem/sec.)"
<< std::endl;
double time_total = time_aggregated + time_merged;
std::cerr
<< "Total in " << time_total
<< " (" << n / time_total << " elem/sec.)"
<< std::endl;
std::cerr << "Size: " << global_map.size() << std::endl << std::endl;
}
/* if (!method || method == 5)
{
*/ /** Вариант 5.
* В разных потоках агрегируем независимо в разные хэш-таблицы,
* пока их размер не станет достаточно большим.
* Если размер локальной хэш-таблицы большой, и в ней нет элемента,
* то вставляем его в одну глобальную хэш-таблицу, содержащую маленькие защёлки в каждой ячейке,
* а если защёлку не удалось захватить, то вставляем в локальную.
* Затем сливаем все локальные хэш-таблицы в глобальную.
*/
/*
Map local_maps[num_threads];
MapSmallLocks global_map;
Stopwatch watch;
for (size_t i = 0; i < num_threads; ++i)
pool.schedule(std::bind(aggregate5,
std::ref(local_maps[i]),
std::ref(global_map),
data.begin() + (data.size() * i) / num_threads,
data.begin() + (data.size() * (i + 1)) / num_threads));
pool.wait();
watch.stop();
double time_aggregated = watch.elapsedSeconds();
std::cerr
<< "Aggregated in " << time_aggregated
<< " (" << n / time_aggregated << " elem/sec.)"
<< std::endl;
size_t size_before_merge = 0;
std::cerr << "Sizes (local): ";
for (size_t i = 0; i < num_threads; ++i)
{
std::cerr << (i == 0 ? "" : ", ") << local_maps[i].size();
size_before_merge += local_maps[i].size();
}
std::cerr << std::endl;
std::cerr << "Size (global): " << global_map.size() << std::endl;
size_before_merge += global_map.size();
watch.restart();
for (size_t i = 0; i < num_threads; ++i)
for (auto it = local_maps[i].begin(); it != local_maps[i].end(); ++it)
global_map.insert(std::make_pair(it->first, 0)).first->second += it->second;
pool.wait();
watch.stop();
double time_merged = watch.elapsedSeconds();
std::cerr
<< "Merged in " << time_merged
<< " (" << size_before_merge / time_merged << " elem/sec.)"
<< std::endl;
double time_total = time_aggregated + time_merged;
std::cerr
<< "Total in " << time_total
<< " (" << n / time_total << " elem/sec.)"
<< std::endl;
std::cerr << "Size: " << global_map.size() << std::endl << std::endl;
}*/
/*if (!method || method == 6)
{
*//** Вариант 6.
* В разных потоках агрегируем независимо в разные хэш-таблицы.
* Затем "сливаем" их, проходя по ним в одинаковом порядке ключей.
* Довольно тормозной вариант.
*/
/*
std::vector<Map> maps(num_threads);
Stopwatch watch;
for (size_t i = 0; i < num_threads; ++i)
pool.schedule(std::bind(aggregate1,
std::ref(maps[i]),
data.begin() + (data.size() * i) / num_threads,
data.begin() + (data.size() * (i + 1)) / num_threads));
pool.wait();
watch.stop();
double time_aggregated = watch.elapsedSeconds();
std::cerr
<< "Aggregated in " << time_aggregated
<< " (" << n / time_aggregated << " elem/sec.)"
<< std::endl;
size_t size_before_merge = 0;
std::cerr << "Sizes: ";
for (size_t i = 0; i < num_threads; ++i)
{
std::cerr << (i == 0 ? "" : ", ") << maps[i].size();
size_before_merge += maps[i].size();
}
std::cerr << std::endl;
watch.restart();
using Maps = std::vector<Map *>;
Maps maps_to_merge(num_threads);
for (size_t i = 0; i < num_threads; ++i)
maps_to_merge[i] = &maps[i];
size_t size = 0;
for (size_t i = 0; i < 100; ++i)
processMergedHashTables(maps_to_merge,
[] (Map::value_type & dst, const Map::value_type & src) { dst.second += src.second; },
[&] (const Map::value_type & dst) { ++size; });
watch.stop();
double time_merged = watch.elapsedSeconds();
std::cerr
<< "Merged in " << time_merged
<< " (" << size_before_merge / time_merged << " elem/sec.)"
<< std::endl;
double time_total = time_aggregated + time_merged;
std::cerr
<< "Total in " << time_total
<< " (" << n / time_total << " elem/sec.)"
<< std::endl;
std::cerr << "Size: " << size << std::endl << std::endl;
}*/
return 0;
}
void test(const Cont &)
{
// Testing if all types are provided.
typename Cont::value_type t0;
typename Cont::reference t1 = t0; CGAL_USE(t1);
typename Cont::const_reference t2 = t0; CGAL_USE(t2);
typename Cont::pointer t3 = &t0;
typename Cont::const_pointer t4 = &t0; CGAL_USE(t4);
typename Cont::size_type t5 = 0; CGAL_USE(t5);
typename Cont::difference_type t6 = t3-t3; CGAL_USE(t6);
typename Cont::iterator t7; CGAL_USE(t7);
typename Cont::const_iterator t8; CGAL_USE(t8);
typename Cont::reverse_iterator t9; CGAL_USE(t9);
typename Cont::const_reverse_iterator t10; CGAL_USE(t10);
typename Cont::allocator_type t15;
std::cout << "Testing empty containers." << std::endl;
Cont c0, c1;
Cont c2(t15);
Cont c3(c2);
Cont c4;
c4 = c2;
typedef std::vector<typename Cont::value_type> Vect;
Vect v0;
const Cont c5(v0.begin(), v0.end());
Cont c6(c5.begin(), c5.end());
typename Cont::allocator_type Al;
Cont c7(c0.begin(), c0.end(), Al);
Cont c8;
c8.insert(c0.rbegin(), c0.rend());
// test conversion iterator-> const_iterator.
typename Cont::const_iterator t16 = c5.begin(); CGAL_USE(t16);
assert(t16 == c5.begin());
assert(c0 == c1);
assert(! (c0 < c1));
assert(check_empty(c0));
assert(check_empty(c1));
assert(check_empty(c2));
assert(check_empty(c3));
assert(check_empty(c4));
assert(check_empty(c5));
assert(check_empty(c6));
assert(check_empty(c7));
assert(check_empty(c8));
c1.swap(c0);
assert(check_empty(c0));
assert(check_empty(c1));
c1.merge(c0);
assert(check_empty(c0));
assert(check_empty(c1));
typename Cont::allocator_type t20 = c0.get_allocator();
std::cout << "Now filling some containers" << std::endl;
Vect v1(10000);
Cont c9(v1.begin(), v1.end());
assert(c9.size() == v1.size());
assert(c9.max_size() >= v1.size());
assert(c9.capacity() >= c9.size());
Cont c10 = c9;
assert(c10 == c9);
assert(c10.size() == v1.size());
assert(c10.max_size() >= v1.size());
assert(c10.capacity() >= c10.size());
c9.clear();
assert(check_empty(c9));
assert(c9.capacity() >= c9.size());
assert(c0 == c9);
c9.merge(c10);
c10.swap(c9);
assert(check_empty(c9));
assert(c9.capacity() >= c9.size());
assert(c10.size() == v1.size());
assert(c10.max_size() >= v1.size());
assert(c10.capacity() >= c10.size());
std::cout << "Testing insertion methods" << std::endl;
c9.assign(c10.begin(), c10.end());
assert(c9 == c10);
c10.assign(c9.begin(), c9.end());
assert(c9 == c10);
c9.insert(c10.begin(), c10.end());
assert(c9.size() == 2*v1.size());
c9.clear();
assert(c9 != c10);
c9.insert(c10.begin(), c10.end());
assert(c9.size() == v1.size());
assert(c9 == c10);
typename Cont::iterator it = c9.iterator_to(*c9.begin());
assert(it == c9.begin());
typename Cont::const_iterator cit = c9.iterator_to(const_cast<typename Cont::const_reference>(*c9.begin()));
assert(cit == c9.begin());
typename Cont::iterator s_it = Cont::s_iterator_to(*c9.begin());
assert(s_it == c9.begin());
typename Cont::const_iterator s_cit = Cont::s_iterator_to(const_cast<typename Cont::const_reference>(*c9.begin()));
assert(s_cit == c9.begin());
c10 = Cont();
assert(check_empty(c10));
for(typename Vect::const_iterator it = v1.begin(); it != v1.end(); ++it)
c10.insert(*it);
assert(c10.size() == v1.size());
assert(c9 == c10);
c9.erase(c9.begin());
c9.erase(c9.begin());
assert(c9.size() == v1.size() - 2);
// test reserve
/*Cont c11;
c11.reserve(v1.size());
for(typename Vect::const_iterator it = v1.begin(); it != v1.end(); ++it)
c11.insert(*it);
assert(c11.size() == v1.size());
assert(c10 == c11);*/
// owns() and owns_dereferencable().
for(typename Cont::const_iterator it = c9.begin(), end = c9.end(); it != end; ++it) {
assert(c9.owns(it));
assert(c9.owns_dereferencable(it));
assert(! c10.owns(it));
assert(! c10.owns_dereferencable(it));
}
assert(c9.owns(c9.end()));
assert(! c9.owns_dereferencable(c9.end()));
c9.erase(c9.begin(), c9.end());
assert(check_empty(c9));
std::cout << "Testing parallel insertion" << std::endl;
{
Cont c11;
Vect v11(1000000);
std::vector<typename Cont::iterator> iterators(v11.size());
tbb::parallel_for(
tbb::blocked_range<size_t>( 0, v11.size() ),
Insert_in_CCC_functor<Vect, Cont>(v11, c11, iterators)
);
assert(c11.size() == v11.size());
std::cout << "Testing parallel erasure" << std::endl;
tbb::parallel_for(
tbb::blocked_range<size_t>( 0, v11.size() ),
Erase_in_CCC_functor<Cont>(c11, iterators)
);
assert(c11.empty());
}
std::cout << "Testing parallel insertion AND erasure" << std::endl;
{
Cont c12;
Vect v12(1000000);
std::vector<tbb::atomic<bool> > free_elements(v12.size());
for(typename std::vector<tbb::atomic<bool> >::iterator
it = free_elements.begin(), end = free_elements.end(); it != end; ++it)
{
*it = true;
}
tbb::atomic<unsigned int> num_erasures;
num_erasures = 0;
std::vector<typename Cont::iterator> iterators(v12.size());
tbb::parallel_for(
tbb::blocked_range<size_t>( 0, v12.size() ),
Insert_and_erase_in_CCC_functor<Vect, Cont>(
v12, c12, iterators, free_elements, num_erasures)
);
assert(c12.size() == v12.size() - num_erasures);
}
}
int
run_program(int argc, char **argv)
{
CommandLineArgument<int> number_of_list_files;
int next_command_line_argument = 0;
bool echo_commands = false;
for (int i = 1; ((i < argc) && (!have_argument_p(number_of_list_files))); i++) {
std::string argument(argv[i]);
if (argument == "--help") {
print_usage();
return 0;
} else if (argument == "--echo") {
echo_commands = true;
} else if (!have_argument_p(number_of_list_files)) {
number_of_list_files = argument;
next_command_line_argument = i + 1;
}
}
if (!have_argument_p(number_of_list_files)) {
print_usage();
return -1;
}
if (*number_of_list_files == 0)
return 0;
const int minimum_number_of_arguments = 0
+ *number_of_list_files
+ 1 // the command;
+ 0;
if ((argc - next_command_line_argument) < minimum_number_of_arguments)
throw make_runtime_error("Not enough arguments available for processing.\n");
std::list<std::string> standard_input_list;
bool have_read_standard_input_list = false;
std::vector<std::string> list_filenames(*number_of_list_files);
std::vector<std::list<std::string> > lists(*number_of_list_files);
for (int i = 0; i < *number_of_list_files; i++) {
int argument_index = i + next_command_line_argument;
std::string list_file = argv[argument_index];
std::list<std::string> l;
if ((list_file == "-") && (have_read_standard_input_list)) {
l = standard_input_list;
} else if ((list_file == "-")) {
standard_input_list = read_list(std::cin);
have_read_standard_input_list = true;
l = standard_input_list;
} else if (!file_exists_p(list_file)) {
throw make_runtime_error("List file %s does not exist.", list_file.c_str());
} else {
l = read_list(list_file.c_str());
}
list_filenames[i] = list_file;
lists[i] = l;
}
next_command_line_argument += *number_of_list_files;
// read the command and its options.
std::string command(argv[next_command_line_argument]);
std::list<std::string> command_arguments;
std::copy(argv + next_command_line_argument + 1, argv + argc, std::back_inserter(command_arguments));
// check all lists are the same size.
for (size_t i = 0; i < lists.size(); i++) {
if (lists[i].size() != lists[0].size())
throw make_runtime_error("The number of entires in list %s (%d) differs to %s (%d).",
list_filenames[i].c_str(),
lists[i].size(),
list_filenames[0].c_str(),
lists[0].size());
}
EchoRunner echo_runner(std::cout);
ForkRunner fork_runner;
Runner *runner = 0;
if (echo_commands)
runner = &echo_runner;
else
runner = &fork_runner;
assert(runner);
std::vector<std::list<std::string>::const_iterator> iterators(*number_of_list_files);
for (int i = 0; i < *number_of_list_files; i++)
iterators[i] = lists[i].begin();
for (size_t i = 0; i < lists[0].size(); i++) {
std::string invocation_command = command;
std::list<std::string> invocation_arguments;
std::copy(command_arguments.begin(), command_arguments.end(), std::back_inserter(invocation_arguments));
for (size_t j = 0; j < iterators.size(); j++) {
invocation_arguments.push_back(*iterators[j]);
iterators[j]++;
}
runner->perform(invocation_command, invocation_arguments);
}
return 0;
}