/* * Wait for termination */ void wait_end_tasks(int rank) { int bz; int nbz = get_nbz(); for (bz = 0; bz < nbz; bz++) { if (get_block_mpi_node(bz) == rank) { /* Wait for the task producing block "bz" */ starpu_tag_wait(TAG_FINISH(bz)); /* Get the result back to memory */ struct block_description *block = get_block_description(bz); starpu_data_acquire(block->layers_handle[0], STARPU_R); starpu_data_acquire(block->layers_handle[1], STARPU_R); /* the data_acquire here is done to make sure * the data is sent back to the ram memory, we * can safely do a data_release, to avoid the * data_unregister to block later on */ starpu_data_release(block->layers_handle[0]); starpu_data_release(block->layers_handle[1]); } } }
/* R(z) = R(z+d) = local, just call the save kernel */ static void create_task_save_local(unsigned iter, unsigned z, int dir, int local_rank) { struct starpu_task *save_task = starpu_task_create(); struct block_description *descr = get_block_description(z); save_task->cl = (dir == -1)?&save_cl_bottom:&save_cl_top; save_task->cl_arg = descr; /* Saving our border... */ save_task->handles[0] = descr->layers_handle[0]; save_task->handles[1] = descr->layers_handle[1]; /* ... to the neighbour's copy */ struct block_description *neighbour = descr->boundary_blocks[(1+dir)/2]; save_task->handles[2] = neighbour->boundaries_handle[(1-dir)/2][0]; save_task->handles[3] = neighbour->boundaries_handle[(1-dir)/2][1]; /* Bind */ if (iter <= BIND_LAST) save_task->execute_on_a_specific_worker = get_bind_tasks(); save_task->workerid = descr->preferred_worker; int ret = starpu_task_submit(save_task); if (ret) { FPRINTF(stderr, "Could not submit task save: %d\n", ret); STARPU_ABORT(); } }
WT_Boolean WT_Block_Meaning::operator== (WT_Block_Meaning const & meaning) const { if(get_block_description() == meaning.get_block_description()) return WD_True; return WD_False; }
WT_Result WT_Block_Meaning::serialize(WT_File & file) const { WD_CHECK (file.dump_delayed_drawable()); if (file.heuristics().allow_binary_data()) { WD_CHECK (file.write((WT_Byte) '{')); WD_CHECK (file.write((WT_Integer32) (sizeof(WT_Unsigned_Integer16) + // for the opcode sizeof(WT_Unsigned_Integer16) + // block description sizeof(WT_Byte) // The closing "}" ))); WD_CHECK (file.write((WT_Unsigned_Integer16) WD_EXBO_BLOCK_MEANING)); WD_CHECK (file.write((WT_Unsigned_Integer16) get_block_description())); WD_CHECK (file.write((WT_Byte) '}')); } else { //Extended ASCII format WD_CHECK (file.write_geom_tab_level()); WD_CHECK (file.write("(BlockMeaning ")); if(get_block_description() == WT_Block_Meaning::None) { WD_CHECK (file.write_quoted_string("None ", WD_True)); } else if (get_block_description() == WT_Block_Meaning::Seal) { WD_CHECK (file.write_quoted_string("Seal ", WD_True)); } else if (get_block_description() == WT_Block_Meaning::Stamp) { WD_CHECK (file.write_quoted_string("Stamp ", WD_True)); } else if (get_block_description() == WT_Block_Meaning::Label) { WD_CHECK (file.write_quoted_string("Label ", WD_True)); } else if (get_block_description() == WT_Block_Meaning::Redline) { WD_CHECK (file.write_quoted_string("Redline ", WD_True)); } else if (get_block_description() == WT_Block_Meaning::Reserved1) { WD_CHECK (file.write_quoted_string("Reserved1", WD_True)); } else if (get_block_description() == WT_Block_Meaning::Reserved2) { WD_CHECK (file.write_quoted_string("Reserved2", WD_True)); } WD_CHECK (file.write(")")); } return WT_Result::Success; }
/* Post MPI send */ static void create_task_save_mpi_send(unsigned iter, unsigned z, int dir, int local_rank) { struct block_description *descr = get_block_description(z); STARPU_ASSERT(descr->mpi_node == local_rank); struct block_description *neighbour = descr->boundary_blocks[(1+dir)/2]; int dest = neighbour->mpi_node; STARPU_ASSERT(neighbour->mpi_node != local_rank); /* Send neighbour's border copy to the neighbour */ starpu_data_handle_t handle0 = neighbour->boundaries_handle[(1-dir)/2][0]; starpu_data_handle_t handle1 = neighbour->boundaries_handle[(1-dir)/2][1]; starpu_mpi_isend_detached(handle0, dest, MPI_TAG0(z, iter, dir), MPI_COMM_WORLD, send_done, (void*)(uintptr_t)z); starpu_mpi_isend_detached(handle1, dest, MPI_TAG1(z, iter, dir), MPI_COMM_WORLD, send_done, (void*)(uintptr_t)z); }
/* Post MPI recv */ static void create_task_save_mpi_recv(unsigned iter, unsigned z, int dir, int local_rank) { struct block_description *descr = get_block_description(z); STARPU_ASSERT(descr->mpi_node != local_rank); struct block_description *neighbour = descr->boundary_blocks[(1+dir)/2]; int source = descr->mpi_node; STARPU_ASSERT(neighbour->mpi_node == local_rank); /* Receive our neighbour's border in our neighbour copy */ starpu_data_handle_t handle0 = neighbour->boundaries_handle[(1-dir)/2][0]; starpu_data_handle_t handle1 = neighbour->boundaries_handle[(1-dir)/2][1]; starpu_mpi_irecv_detached(handle0, source, MPI_TAG0(z, iter, dir), MPI_COMM_WORLD, recv_done, (void*)(uintptr_t)z); starpu_mpi_irecv_detached(handle1, source, MPI_TAG1(z, iter, dir), MPI_COMM_WORLD, recv_done, (void*)(uintptr_t)z); }
void create_task_update(unsigned iter, unsigned z, int local_rank) { STARPU_ASSERT(iter != 0); struct starpu_task *task = starpu_task_create(); unsigned niter = get_niter(); /* We are going to synchronize with the last tasks */ if (iter == niter) { task->use_tag = 1; task->tag_id = TAG_FINISH(z); } unsigned old_layer = (K*(iter-1)) % 2; unsigned new_layer = (old_layer + 1) % 2; struct block_description *descr = get_block_description(z); task->handles[0] = descr->layers_handle[new_layer]; task->handles[1] = descr->layers_handle[old_layer]; task->handles[2] = descr->boundaries_handle[T][new_layer]; task->handles[3] = descr->boundaries_handle[T][old_layer]; task->handles[4] = descr->boundaries_handle[B][new_layer]; task->handles[5] = descr->boundaries_handle[B][old_layer]; task->cl = &cl_update; task->cl_arg = descr; if (iter <= BIND_LAST) task->execute_on_a_specific_worker = get_bind_tasks(); task->workerid = descr->preferred_worker; int ret = starpu_task_submit(task); if (ret) { FPRINTF(stderr, "Could not submit task update block: %d\n", ret); if (ret == -ENODEV) exit(77); STARPU_ABORT(); } }
void create_start_task(int z, int dir) { /* Dumb task depending on the init task and simulating writing the neighbour buffers, to avoid communications and computation running before we start measuring time */ struct starpu_task *wait_init = starpu_task_create(); struct block_description *descr = get_block_description(z); starpu_tag_t tag_init = TAG_INIT_TASK; wait_init->cl = &null; wait_init->use_tag = 1; wait_init->tag_id = TAG_START(z, dir); wait_init->handles[0] = descr->boundaries_handle[(1 + dir) / 2][0]; wait_init->handles[1] = descr->boundaries_handle[(1 + dir) / 2][1]; starpu_tag_declare_deps_array(wait_init->tag_id, 1, &tag_init); int ret = starpu_task_submit(wait_init); if (ret) { FPRINTF(stderr, "Could not submit task initial wait: %d\n", ret); STARPU_ABORT(); } }