size_t size() {
auto self = this->self;
call_on_all_cores([self]{ self->count = 0; });
forall_keys([self](K& k){ self->count++; });
on_all_cores([self]{ self->count = allreduce<size_t,collective_add>(self->count); });
return count;
}
/// Enroll more things that need to be completed before the global completion is, well, complete.
/// This will send a cancel to the master if this core previously entered the cancellable barrier.
///
/// Blocks until cancel completes (if it must cancel) to ensure correct ordering, therefore
/// cannot be called from message handler.
void enroll(int64_t inc = 1) {
if (inc == 0) return;
CHECK_GE(inc, 1);
count += inc;
DVLOG(5) << "enroll " << inc << " -> " << count << " gce("<<this<<")";
// first one to have work here
if (count == inc) { // count[0 -> inc]
event_in_progress = true; // optimization to save checking in wait()
// cancel barrier
Core co = impl::call(master_core, [this] {
cores_out++;
return cores_out;
});
// first one to cancel barrier should make sure other cores are ready to wait
if (co == 1) { // cores_out[0 -> 1]
event_in_progress = true;
call_on_all_cores([this] {
event_in_progress = true;
});
CHECK(event_in_progress);
}
// block until cancelled
CHECK_GT(count, 0);
DVLOG(2) << "gce(" << this << " cores_out: " << co << ", count: " << count << ")";
}
}
void clear() {
auto b = self;
call_on_all_cores([=]{
for (T& e : util::iterate(b->l_storage, b->l_size)) e.~T();
b->l_size = 0;
});
}
void destroy() {
auto self = this->self;
forall(this->base, this->capacity, [](Cell& c){ c.~Cell(); });
global_free(this->base);
call_on_all_cores([self]{ self->~GlobalHashSet(); });
global_free(self);
}
static GlobalAddress<GlobalHashSet> create(size_t total_capacity) {
auto base = global_alloc<Cell>(total_capacity);
auto self = symmetric_global_alloc<GlobalHashSet>();
call_on_all_cores([self,base,total_capacity]{
new (self.localize()) GlobalHashSet(self, base, total_capacity);
});
forall(base, total_capacity, [](int64_t i, Cell& c) { new (&c) Cell(); });
return self;
}
static GlobalAddress<GlobalBag> create(size_t total_capacity) {
auto self = symmetric_global_alloc<GlobalBag>();
auto n = total_capacity / cores()
+ total_capacity % cores();
call_on_all_cores([=]{
new (self.localize()) GlobalBag(self, n);
});
return self;
}
void reset_all_cores() {
#ifdef HISTOGRAM_SAMPLED
try {
char * jobid = getenv("SLURM_JOB_ID");
char dir[256]; sprintf(dir, "histogram.%s", jobid);
fs::create_directories(dir);
fs::permissions(dir, fs::perms::all_all);
} catch (fs::filesystem_error& e) {
LOG(ERROR) << "filesystem error: " << e.what();
}
#endif
call_on_all_cores([]{
reset();
});
}
void destroy() {
auto self = this->self;
call_on_all_cores([self]{ self->~GlobalBag(); });
global_free(self);
}
void bfs(GlobalAddress<G> _g, int nbfs, TupleGraph tg) {
bool verified = false;
double t;
auto _frontier = GlobalBag<VertexID>::create(_g->nv);
auto _next = GlobalBag<VertexID>::create(_g->nv);
call_on_all_cores([=]{ frontier = _frontier; next = _next; g = _g; });
// do BFS from multiple different roots and average their times
for (int root_idx = 0; root_idx < nbfs; root_idx++) {
// intialize parent to -1
forall(g, [](G::Vertex& v){ v->init(); v->level = -1; });
VertexID root;
if (FLAGS_max_degree_source) {
forall(g, [](VertexID i, G::Vertex& v){
max_degree << MaxDegree(i, v.nadj);
});
root = static_cast<MaxDegree>(max_degree).idx();
} else {
root = choose_root(g);
}
// setup 'root' as the parent of itself
delegate::call(g->vs+root, [=](G::Vertex& v){
v->parent = root;
v->level = 0;
});
// reset frontier queues
next->clear();
frontier->clear();
// start with root as only thing in frontier
delegate::call((g->vs+root).core(), [=]{ frontier->add(root); });
t = walltime();
bool top_down = true;
int64_t prev_nf = -1;
int64_t frontier_edges = 0;
int64_t remaining_edges = g->nadj;
while (!frontier->empty()) {
auto nf = frontier->size();
VLOG(1) << "remaining_edges = " << remaining_edges << ", nf = " << nf << ", prev_nf = " << prev_nf << ", frontier_edges: " ;
if (top_down && frontier_edges > remaining_edges/FLAGS_beamer_alpha && nf > prev_nf) {
VLOG(1) << "switching to bottom-up";
top_down = false;
} else if (!top_down && frontier_edges < g->nv/FLAGS_beamer_beta && nf < prev_nf) {
VLOG(1) << "switching to top-down";
top_down = true;
}
edge_count = 0;
if (top_down) {
// iterate over vertices in this level of the frontier
forall(frontier, [](VertexID& i){
// visit all the adjacencies of the vertex
// note: this has to be 'async' to prevent deadlock from
// running out of available workers
forall<async>(adj(g,i), [i](G::Edge& e) {
auto j = e.id;
// at the core where the vertex is...
delegate::call<async>(e.ga, [i,j](G::Vertex& vj){
// note: no synchronization needed because 'call' is
// guaranteed to be executed atomically because it
// does no blocking operations
if (vj->parent == -1) {
// claim parenthood
vj->parent = i;
vj->level = current_depth;
next->add(j);
edge_count += vj.nadj;
}
});
});
});
} else { // bottom-up
forall<&phaser>(g, [](G::Vertex& v){
if (v->level != -1) return;
auto va = make_linear(&v);
forall<async,&phaser>(adj(g,v), [=,&v](G::Edge& e){
if (v->level != -1) return;
phaser.enroll();
auto eva = e.ga;
send_heap_message(eva.core(), [=]{
auto& ev = *eva.pointer();
if (ev->level != -1 && ev->level < current_depth) {
auto eid = g->id(ev);
send_heap_message(va.core(), [=]{
auto& v = *va.pointer();
if (v->level == -1) {
next->add(g->id(v));
v->level = current_depth;
v->parent = eid;
edge_count += v.nadj;
}
phaser.complete();
});
} else {
phaser.send_completion(va.core());
}
});
});
});
}
call_on_all_cores([=]{
current_depth++;
// switch to next frontier level
std::swap(frontier, next);
});
next->clear();
frontier_edges = edge_count;
remaining_edges -= frontier_edges;
prev_nf = nf;
} // while (frontier not empty)
double this_bfs_time = walltime() - t;
LOG(INFO) << "(root=" << root << ", time=" << this_bfs_time << ")";
if (!verified) {
// only verify the first one to save time
t = walltime();
bfs_nedge = verify(tg, g, root);
verify_time = (walltime()-t);
LOG(INFO) << verify_time;
verified = true;
Metrics::reset_all_cores(); // don't count the first one
} else {
total_time += this_bfs_time;
}
bfs_mteps += bfs_nedge / this_bfs_time / 1.0e6;
}
}
void stop_tracing() {
call_on_all_cores([]{
stop_tracing_here();
});
}
static void run_sync(GlobalAddress<Graph<V,E>> _g) {
call_on_all_cores([=]{ g = _g; });
ct = 0;
// initialize GraphlabVertexProgram
forall(g, [=](Vertex& v){
v->prog = new VertexProg(v);
if (prog(v).gather_edges(v)) ct++;
});
if (ct > 0) {
forall(g, [=](Vertex& v){
forall<async>(adj(g,v), [=,&v](Edge& e){
// gather
auto delta = prog(v).gather(v, e);
call<async>(e.ga, [=](Vertex& ve){
prog(ve).post_delta(delta);
});
});
});
}
int iteration = 0;
size_t active = V::total_active;
while ( active > 0 && iteration < FLAGS_max_iterations )
GRAPPA_TIME_REGION(iteration_time) {
VLOG(1) << "iteration " << std::setw(3) << iteration;
VLOG(1) << " active: " << active;
double t = walltime();
forall(g, [=](Vertex& v){
if (!v->active) return;
v->deactivate();
auto& p = prog(v);
// apply
p.apply(v, p.cache);
v->active_minor_step = p.scatter_edges(v);
});
forall(g, [=](Vertex& v){
if (v->active_minor_step) {
v->active_minor_step = false;
auto prog_copy = prog(v);
// scatter
forall<async>(adj(g,v), [=](Edge& e){
_do_scatter(prog_copy, e, &VertexProg::scatter);
});
}
});
{
symmetric_static std::ofstream myFile;
//std::ofstream myFile;
int pid = getpid();
LOG(INFO) << "start writing file";
std::string path = NaiveGraphlabEngine<G,VertexProg>::OutputPath;
//on_all_cores( [pid, iteration, path] {
std::ostringstream oss;
oss << OutputPath << "-" << pid << "-" << mycore() << "-" << iteration;
new (&myFile) std::ofstream(oss.str());
if (!myFile.is_open()) exit(1);
//});
forall(g, [](VertexID i, Vertex& v){
// LOG(INFO) << "id: " << i << " label: " << v->label;
myFile << i << " ";
for (int j = 0; j < NaiveGraphlabEngine<G,VertexProg>::Number_of_groups; j++) {
myFile << prog(v).cache.label_count[j] << " ";
}
myFile << v->label << "\n";
});
//on_all_cores( [] {
myFile.close();
//});
LOG(INFO) << "end writig file";
}
iteration++;
VLOG(1) << " time: " << walltime()-t;
active = V::total_active;
}
forall(g, [](Vertex& v){ delete static_cast<VertexProg*>(v->prog); });
}
