/*
* Wait for termination
*/
void wait_end_tasks(int rank)
{
int bz;
int nbz = get_nbz();
for (bz = 0; bz < nbz; bz++)
{
if (get_block_mpi_node(bz) == rank)
{
/* Wait for the task producing block "bz" */
starpu_tag_wait(TAG_FINISH(bz));
/* Get the result back to memory */
struct block_description *block = get_block_description(bz);
starpu_data_acquire(block->layers_handle[0], STARPU_R);
starpu_data_acquire(block->layers_handle[1], STARPU_R);
/* the data_acquire here is done to make sure
* the data is sent back to the ram memory, we
* can safely do a data_release, to avoid the
* data_unregister to block later on
*/
starpu_data_release(block->layers_handle[0]);
starpu_data_release(block->layers_handle[1]);
}
}
}
/* R(z) = R(z+d) = local, just call the save kernel */
static void create_task_save_local(unsigned iter, unsigned z, int dir, int local_rank)
{
struct starpu_task *save_task = starpu_task_create();
struct block_description *descr = get_block_description(z);
save_task->cl = (dir == -1)?&save_cl_bottom:&save_cl_top;
save_task->cl_arg = descr;
/* Saving our border... */
save_task->handles[0] = descr->layers_handle[0];
save_task->handles[1] = descr->layers_handle[1];
/* ... to the neighbour's copy */
struct block_description *neighbour = descr->boundary_blocks[(1+dir)/2];
save_task->handles[2] = neighbour->boundaries_handle[(1-dir)/2][0];
save_task->handles[3] = neighbour->boundaries_handle[(1-dir)/2][1];
/* Bind */
if (iter <= BIND_LAST)
save_task->execute_on_a_specific_worker = get_bind_tasks();
save_task->workerid = descr->preferred_worker;
int ret = starpu_task_submit(save_task);
if (ret)
{
FPRINTF(stderr, "Could not submit task save: %d\n", ret);
STARPU_ABORT();
}
}
WT_Boolean WT_Block_Meaning::operator== (WT_Block_Meaning
const & meaning) const
{
if(get_block_description() == meaning.get_block_description())
return WD_True;
return WD_False;
}
WT_Result WT_Block_Meaning::serialize(WT_File & file) const
{
WD_CHECK (file.dump_delayed_drawable());
if (file.heuristics().allow_binary_data())
{
WD_CHECK (file.write((WT_Byte) '{'));
WD_CHECK (file.write((WT_Integer32) (sizeof(WT_Unsigned_Integer16) + // for the opcode
sizeof(WT_Unsigned_Integer16) + // block description
sizeof(WT_Byte) // The closing "}"
)));
WD_CHECK (file.write((WT_Unsigned_Integer16) WD_EXBO_BLOCK_MEANING));
WD_CHECK (file.write((WT_Unsigned_Integer16) get_block_description()));
WD_CHECK (file.write((WT_Byte) '}'));
}
else {
//Extended ASCII format
WD_CHECK (file.write_geom_tab_level());
WD_CHECK (file.write("(BlockMeaning "));
if(get_block_description() == WT_Block_Meaning::None) {
WD_CHECK (file.write_quoted_string("None ", WD_True));
}
else if (get_block_description() == WT_Block_Meaning::Seal) {
WD_CHECK (file.write_quoted_string("Seal ", WD_True));
}
else if (get_block_description() == WT_Block_Meaning::Stamp) {
WD_CHECK (file.write_quoted_string("Stamp ", WD_True));
}
else if (get_block_description() == WT_Block_Meaning::Label) {
WD_CHECK (file.write_quoted_string("Label ", WD_True));
}
else if (get_block_description() == WT_Block_Meaning::Redline) {
WD_CHECK (file.write_quoted_string("Redline ", WD_True));
}
else if (get_block_description() == WT_Block_Meaning::Reserved1) {
WD_CHECK (file.write_quoted_string("Reserved1", WD_True));
}
else if (get_block_description() == WT_Block_Meaning::Reserved2) {
WD_CHECK (file.write_quoted_string("Reserved2", WD_True));
}
WD_CHECK (file.write(")"));
}
return WT_Result::Success;
}
/* Post MPI send */
static void create_task_save_mpi_send(unsigned iter, unsigned z, int dir, int local_rank)
{
struct block_description *descr = get_block_description(z);
STARPU_ASSERT(descr->mpi_node == local_rank);
struct block_description *neighbour = descr->boundary_blocks[(1+dir)/2];
int dest = neighbour->mpi_node;
STARPU_ASSERT(neighbour->mpi_node != local_rank);
/* Send neighbour's border copy to the neighbour */
starpu_data_handle_t handle0 = neighbour->boundaries_handle[(1-dir)/2][0];
starpu_data_handle_t handle1 = neighbour->boundaries_handle[(1-dir)/2][1];
starpu_mpi_isend_detached(handle0, dest, MPI_TAG0(z, iter, dir), MPI_COMM_WORLD, send_done, (void*)(uintptr_t)z);
starpu_mpi_isend_detached(handle1, dest, MPI_TAG1(z, iter, dir), MPI_COMM_WORLD, send_done, (void*)(uintptr_t)z);
}
/* Post MPI recv */
static void create_task_save_mpi_recv(unsigned iter, unsigned z, int dir, int local_rank)
{
struct block_description *descr = get_block_description(z);
STARPU_ASSERT(descr->mpi_node != local_rank);
struct block_description *neighbour = descr->boundary_blocks[(1+dir)/2];
int source = descr->mpi_node;
STARPU_ASSERT(neighbour->mpi_node == local_rank);
/* Receive our neighbour's border in our neighbour copy */
starpu_data_handle_t handle0 = neighbour->boundaries_handle[(1-dir)/2][0];
starpu_data_handle_t handle1 = neighbour->boundaries_handle[(1-dir)/2][1];
starpu_mpi_irecv_detached(handle0, source, MPI_TAG0(z, iter, dir), MPI_COMM_WORLD, recv_done, (void*)(uintptr_t)z);
starpu_mpi_irecv_detached(handle1, source, MPI_TAG1(z, iter, dir), MPI_COMM_WORLD, recv_done, (void*)(uintptr_t)z);
}
void create_task_update(unsigned iter, unsigned z, int local_rank)
{
STARPU_ASSERT(iter != 0);
struct starpu_task *task = starpu_task_create();
unsigned niter = get_niter();
/* We are going to synchronize with the last tasks */
if (iter == niter)
{
task->use_tag = 1;
task->tag_id = TAG_FINISH(z);
}
unsigned old_layer = (K*(iter-1)) % 2;
unsigned new_layer = (old_layer + 1) % 2;
struct block_description *descr = get_block_description(z);
task->handles[0] = descr->layers_handle[new_layer];
task->handles[1] = descr->layers_handle[old_layer];
task->handles[2] = descr->boundaries_handle[T][new_layer];
task->handles[3] = descr->boundaries_handle[T][old_layer];
task->handles[4] = descr->boundaries_handle[B][new_layer];
task->handles[5] = descr->boundaries_handle[B][old_layer];
task->cl = &cl_update;
task->cl_arg = descr;
if (iter <= BIND_LAST)
task->execute_on_a_specific_worker = get_bind_tasks();
task->workerid = descr->preferred_worker;
int ret = starpu_task_submit(task);
if (ret)
{
FPRINTF(stderr, "Could not submit task update block: %d\n", ret);
if (ret == -ENODEV)
exit(77);
STARPU_ABORT();
}
}
void create_start_task(int z, int dir)
{
/* Dumb task depending on the init task and simulating writing the
neighbour buffers, to avoid communications and computation running
before we start measuring time */
struct starpu_task *wait_init = starpu_task_create();
struct block_description *descr = get_block_description(z);
starpu_tag_t tag_init = TAG_INIT_TASK;
wait_init->cl = &null;
wait_init->use_tag = 1;
wait_init->tag_id = TAG_START(z, dir);
wait_init->handles[0] = descr->boundaries_handle[(1 + dir) / 2][0];
wait_init->handles[1] = descr->boundaries_handle[(1 + dir) / 2][1];
starpu_tag_declare_deps_array(wait_init->tag_id, 1, &tag_init);
int ret = starpu_task_submit(wait_init);
if (ret)
{
FPRINTF(stderr, "Could not submit task initial wait: %d\n", ret);
STARPU_ABORT();
}
}
