void icetDestroyContext(IceTContext context)
{
IceTContext saved_current_context;
saved_current_context = icetGetContext();
if (context == saved_current_context) {
icetRaiseDebug("Destroying current context.");
saved_current_context = NULL;
}
/* Temporarily make the context to be destroyed current. */
icetSetContext(context);
/* Call destructors for other dependent units. */
callDestructor(ICET_RENDER_LAYER_DESTRUCTOR);
/* From here on out be careful. We are invalidating the context. */
context->magic_number = 0;
icetStateDestroy(context->state);
context->state = NULL;
context->communicator->Destroy(context->communicator);
/* The context is now completely destroyed and now null. Restore saved
context. */
icetSetContext(saved_current_context);
}
static IceTImage drawInvokeStrategy(void)
{
IceTImage image;
IceTVoid *value;
IceTEnum strategy;
IceTInt display_tile;
IceTInt valid_tile;
icetGetPointerv(ICET_DRAW_FUNCTION, &value);
if (value == NULL) {
icetRaiseError("Drawing function not set. Call icetDrawCallback.",
ICET_INVALID_OPERATION);
return icetImageNull();
}
icetRaiseDebug("Calling strategy");
icetStateSetBoolean(ICET_IS_DRAWING_FRAME, 1);
icetGetEnumv(ICET_STRATEGY, &strategy);
image = icetInvokeStrategy(strategy);
icetStateSetBoolean(ICET_IS_DRAWING_FRAME, 0);
/* Ensure that the returned image is the expected size. */
icetGetIntegerv(ICET_VALID_PIXELS_TILE, &valid_tile);
icetGetIntegerv(ICET_TILE_DISPLAYED, &display_tile);
if ((valid_tile != display_tile) && icetIsEnabled(ICET_COLLECT_IMAGES)) {
icetRaiseDebug2("Display tile: %d, valid tile: %d",
display_tile, valid_tile);
icetRaiseError("Got unexpected tile from strategy.",
ICET_SANITY_CHECK_FAIL);
}
if (valid_tile >= 0) {
const IceTInt *valid_tile_viewport
= icetUnsafeStateGetInteger(ICET_TILE_VIEWPORTS) + 4*valid_tile;
if ( (valid_tile_viewport[2] != icetImageGetWidth(image))
|| (valid_tile_viewport[3] != icetImageGetHeight(image)) ) {
IceTInt valid_offset;
IceTInt valid_num;
icetRaiseDebug1("Tile returned from strategy: %d\n", valid_tile);
icetRaiseDebug4("Expected size: %d %d. Returned size: %d %d",
valid_tile_viewport[2], valid_tile_viewport[3],
(int)icetImageGetWidth(image),
(int)icetImageGetHeight(image));
icetGetIntegerv(ICET_VALID_PIXELS_OFFSET, &valid_offset);
icetGetIntegerv(ICET_VALID_PIXELS_NUM, &valid_num);
icetRaiseDebug2("Reported pixel offset: %d. Reported pixel count: %d",
valid_offset, valid_num);
icetRaiseError("Got unexpected image size from strategy.",
ICET_SANITY_CHECK_FAIL);
}
}
icetStateCheckMemory();
return image;
}
void gl_destroy(void)
{
IceTInt icet_texture;
GLuint gl_texture;
icetRaiseDebug("In OpenGL layer destructor.");
icetGetIntegerv(ICET_GL_INFLATE_TEXTURE, &icet_texture);
gl_texture = icet_texture;
if (gl_texture != 0) {
glDeleteTextures(1, &gl_texture);
}
icetStateSetInteger(ICET_GL_INFLATE_TEXTURE, 0);
}
static void drawCollectTileInformation(void)
{
IceTBoolean *all_contained_masks;
IceTInt num_proc;
IceTInt num_tiles;
icetGetIntegerv(ICET_NUM_PROCESSES, &num_proc);
icetGetIntegerv(ICET_NUM_TILES, &num_tiles);
all_contained_masks
= icetStateAllocateBoolean(ICET_ALL_CONTAINED_TILES_MASKS,
num_tiles*num_proc);
icetRaiseDebug("Gathering rendering information.");
{
const IceTBoolean *contained_mask;
contained_mask = icetUnsafeStateGetBoolean(ICET_CONTAINED_TILES_MASK);
icetCommAllgather(contained_mask, num_tiles, ICET_BYTE,
all_contained_masks);
}
{
IceTInt *contrib_counts;
IceTInt total_image_count;
IceTInt tile_id;
contrib_counts
= icetStateAllocateInteger(ICET_TILE_CONTRIB_COUNTS, num_tiles);
total_image_count = 0;
for (tile_id = 0; tile_id < num_tiles; tile_id++) {
IceTInt proc_id;
contrib_counts[tile_id] = 0;
for (proc_id = 0; proc_id < num_proc; proc_id++) {
if (all_contained_masks[proc_id*num_tiles + tile_id]) {
contrib_counts[tile_id]++;
}
}
total_image_count += contrib_counts[tile_id];
}
icetStateSetIntegerv(ICET_TOTAL_IMAGE_COUNT, 1, &total_image_count);
}
}
void icetDestroyContext(IceTContext context)
{
struct IceTContext *cp = &(context_list[context]);
if (context == current_context_index) {
icetRaiseDebug("Destroying current context.");
}
icetStateDestroy(cp->state);
cp->state = NULL;
free(cp->buffer);
cp->communicator->Destroy(cp->communicator);
cp->buffer = NULL;
cp->buffer_size = 0;
cp->buffer_offset = 0;
if (cp->display_inflate_texture != 0) {
glDeleteTextures(1, &(cp->display_inflate_texture));
}
}
IceTImage icetDirectCompose(void)
{
IceTImage image;
IceTVoid *inSparseImageBuffer;
IceTSparseImage outSparseImage;
IceTSizeType sparseImageSize;
const IceTInt *contrib_counts;
const IceTInt *display_nodes;
IceTInt max_width, max_height;
IceTInt num_tiles;
IceTInt num_contributors;
IceTInt display_tile;
IceTInt tile;
IceTInt *tile_image_dest;
icetRaiseDebug("In Direct Compose");
icetGetIntegerv(ICET_TILE_MAX_WIDTH, &max_width);
icetGetIntegerv(ICET_TILE_MAX_HEIGHT, &max_height);
icetGetIntegerv(ICET_NUM_TILES, &num_tiles);
sparseImageSize = icetSparseImageBufferSize(max_width, max_height);
image = icetGetStateBufferImage(DIRECT_IMAGE_BUFFER,
max_width, max_height);
inSparseImageBuffer = icetGetStateBuffer(DIRECT_IN_SPARSE_IMAGE_BUFFER,
sparseImageSize);
outSparseImage = icetGetStateBufferSparseImage(
DIRECT_OUT_SPARSE_IMAGE_BUFFER,
max_width, max_height);
tile_image_dest = icetGetStateBuffer(DIRECT_TILE_IMAGE_DEST_BUFFER,
num_tiles*sizeof(IceTInt));
icetGetIntegerv(ICET_TILE_DISPLAYED, &display_tile);
if (display_tile >= 0) {
contrib_counts = icetUnsafeStateGetInteger(ICET_TILE_CONTRIB_COUNTS);
num_contributors = contrib_counts[display_tile];
} else {
num_contributors = 0;
}
display_nodes = icetUnsafeStateGetInteger(ICET_DISPLAY_NODES);
for (tile = 0; tile < num_tiles; tile++) {
tile_image_dest[tile] = display_nodes[tile];
}
icetRaiseDebug("Rendering and transferring images.");
icetRenderTransferFullImages(image,
inSparseImageBuffer,
outSparseImage,
tile_image_dest);
if (display_tile >= 0) {
if (num_contributors > 0) {
icetImageCorrectBackground(image);
} else {
/* Must be displaying a blank tile. */
const IceTInt *tile_viewports
= icetUnsafeStateGetInteger(ICET_TILE_VIEWPORTS);
const IceTInt *display_tile_viewport
= tile_viewports + 4*display_tile;
IceTInt display_tile_width = display_tile_viewport[2];
IceTInt display_tile_height = display_tile_viewport[3];
icetRaiseDebug("Returning blank tile.");
icetImageSetDimensions(image,
display_tile_width,
display_tile_height);
icetClearImageTrueBackground(image);
}
}
return image;
}
void icetGLDrawCallbackFunction(const IceTDouble *projection_matrix,
const IceTDouble *modelview_matrix,
const IceTFloat *background_color,
const IceTInt *readback_viewport,
IceTImage result)
{
IceTSizeType width = icetImageGetWidth(result);
IceTSizeType height = icetImageGetHeight(result);
GLint gl_viewport[4];
glGetIntegerv(GL_VIEWPORT, gl_viewport);
/* Check OpenGL state. */
{
if ((gl_viewport[2] != width) || (gl_viewport[3] != height)) {
icetRaiseError("OpenGL viewport different than expected."
" Was it changed?", ICET_SANITY_CHECK_FAIL);
}
}
/* Set up OpenGL. */
{
/* Load the matrices. */
glMatrixMode(GL_PROJECTION);
glLoadMatrixd(projection_matrix);
glMatrixMode(GL_MODELVIEW);
glLoadMatrixd(modelview_matrix);
/* Set the clear color as the background IceT currently wants. */
glClearColor(background_color[0], background_color[1],
background_color[2], background_color[3]);
}
/* Call the rendering callback. */
{
IceTVoid *value;
IceTGLDrawCallbackType callback;
icetRaiseDebug("Calling OpenGL draw function.");
icetGetPointerv(ICET_GL_DRAW_FUNCTION, &value);
callback = (IceTGLDrawCallbackType)value;
(*callback)();
}
/* Temporarily stop render time while reading back buffer. */
icetTimingRenderEnd();
icetTimingBufferReadBegin();
/* Read the OpenGL buffers. */
{
IceTEnum color_format = icetImageGetColorFormat(result);
IceTEnum depth_format = icetImageGetDepthFormat(result);
IceTEnum readbuffer;
IceTSizeType x_offset = gl_viewport[0] + readback_viewport[0];
IceTSizeType y_offset = gl_viewport[1] + readback_viewport[1];
glPixelStorei(GL_PACK_ROW_LENGTH, (GLint)icetImageGetWidth(result));
/* These pixel store parameters are not working on one of the platforms
* I am testing on (thank you Mac). Instead of using these, just offset
* the buffers we read in from. */
/* glPixelStorei(GL_PACK_SKIP_PIXELS, readback_viewport[0]); */
/* glPixelStorei(GL_PACK_SKIP_ROWS, readback_viewport[1]); */
icetGetEnumv(ICET_GL_READ_BUFFER, &readbuffer);
glReadBuffer(readbuffer);
if (color_format == ICET_IMAGE_COLOR_RGBA_UBYTE) {
IceTUInt *colorBuffer = icetImageGetColorui(result);
glReadPixels((GLint)x_offset,
(GLint)y_offset,
(GLsizei)readback_viewport[2],
(GLsizei)readback_viewport[3],
GL_RGBA,
GL_UNSIGNED_BYTE,
colorBuffer + ( readback_viewport[0]
+ width*readback_viewport[1]));
} else if (color_format == ICET_IMAGE_COLOR_RGBA_FLOAT) {
IceTFloat *colorBuffer = icetImageGetColorf(result);
glReadPixels((GLint)x_offset,
(GLint)y_offset,
(GLsizei)readback_viewport[2],
(GLsizei)readback_viewport[3],
GL_RGBA,
GL_FLOAT,
colorBuffer + 4*( readback_viewport[0]
+ width*readback_viewport[1]));
} else if (color_format != ICET_IMAGE_COLOR_NONE) {
icetRaiseError("Invalid color format.", ICET_SANITY_CHECK_FAIL);
}
if (depth_format == ICET_IMAGE_DEPTH_FLOAT) {
IceTFloat *depthBuffer = icetImageGetDepthf(result);;
glReadPixels((GLint)x_offset,
(GLint)y_offset,
(GLsizei)readback_viewport[2],
(GLsizei)readback_viewport[3],
GL_DEPTH_COMPONENT,
GL_FLOAT,
depthBuffer + ( readback_viewport[0]
+ width*readback_viewport[1]));
} else if (depth_format != ICET_IMAGE_DEPTH_NONE) {
icetRaiseError("Invalid depth format.", ICET_SANITY_CHECK_FAIL);
}
glPixelStorei(GL_PACK_ROW_LENGTH, 0);
/* glPixelStorei(GL_PACK_SKIP_PIXELS, 0); */
/* glPixelStorei(GL_PACK_SKIP_ROWS, 0); */
}
icetTimingBufferReadEnd();
/* Start render timer again. It's going to be shut off immediately on
return anyway, but the calling function expects it to be running. */
icetTimingRenderBegin();
}
IceTImage icetDrawFrame(const IceTDouble *projection_matrix,
const IceTDouble *modelview_matrix,
const IceTFloat *background_color)
{
IceTInt frame_count;
IceTImage image;
IceTDouble render_time;
IceTDouble buf_read_time;
IceTDouble compose_time;
IceTDouble total_time;
icetRaiseDebug("In icetDrawFrame");
{
IceTBoolean isDrawing;
icetGetBooleanv(ICET_IS_DRAWING_FRAME, &isDrawing);
if (isDrawing) {
icetRaiseError("Recursive frame draw detected.",
ICET_INVALID_OPERATION);
return icetImageNull();
}
}
icetStateResetTiming();
icetTimingDrawFrameBegin();
icetStateSetDoublev(ICET_PROJECTION_MATRIX, 16, projection_matrix);
icetStateSetDoublev(ICET_MODELVIEW_MATRIX, 16, modelview_matrix);
drawUseBackgroundColor(background_color);
icetGetIntegerv(ICET_FRAME_COUNT, &frame_count);
frame_count++;
icetStateSetIntegerv(ICET_FRAME_COUNT, 1, &frame_count);
drawProjectBounds();
{
IceTEnum strategy;
icetGetEnumv(ICET_STRATEGY, &strategy);
/* drawCollectTileInformation does an allgather to get information
* about the tiles in other processes. These variables are
* ICET_ALL_CONTAINED_TILES_MASKS, ICET_TILE_CONTRIB_COUNTS, and
* ICET_TOTAL_IMAGE_COUNT. However, the sequential strategy ignores
* this information and just uses all processes for all tiles. When
* compositing a single tile, this is a fine strategy and we can save
* a significant proportion of frame time by skipping this step. */
if (strategy != ICET_STRATEGY_SEQUENTIAL) {
drawCollectTileInformation();
}
}
{
IceTInt tile_displayed;
icetGetIntegerv(ICET_TILE_DISPLAYED, &tile_displayed);
if (tile_displayed >= 0) {
const IceTInt *tile_viewports
= icetUnsafeStateGetInteger(ICET_TILE_VIEWPORTS);
IceTInt num_pixels = ( tile_viewports[4*tile_displayed+2]
* tile_viewports[4*tile_displayed+3] );
icetStateSetInteger(ICET_VALID_PIXELS_TILE, tile_displayed);
icetStateSetInteger(ICET_VALID_PIXELS_OFFSET, 0);
icetStateSetInteger(ICET_VALID_PIXELS_NUM, num_pixels);
} else {
icetStateSetInteger(ICET_VALID_PIXELS_TILE, -1);
icetStateSetInteger(ICET_VALID_PIXELS_OFFSET, 0);
icetStateSetInteger(ICET_VALID_PIXELS_NUM, 0);
}
}
image = drawInvokeStrategy();
/* Calculate times. */
icetGetDoublev(ICET_RENDER_TIME, &render_time);
icetGetDoublev(ICET_BUFFER_READ_TIME, &buf_read_time);
icetTimingDrawFrameEnd();
icetGetDoublev(ICET_TOTAL_DRAW_TIME, &total_time);
compose_time = total_time - render_time - buf_read_time;
icetStateSetDouble(ICET_COMPOSITE_TIME, compose_time);
icetStateSetDouble(ICET_BUFFER_WRITE_TIME, 0.0);
icetStateCheckMemory();
return image;
}
static IceTImage vtreeCompose(void)
{
GLint rank, num_proc;
GLint num_tiles;
GLint max_pixels;
GLint *display_nodes;
GLint tile_displayed;
GLboolean *all_contained_tmasks;
GLint *tile_viewports;
IceTImage imageBuffer;
IceTSparseImage inImage, outImage;
struct node_info *info;
struct node_info *my_info;
int tile, node;
int tiles_transfered;
int tile_held = -1;
icetRaiseDebug("In vtreeCompose");
/* Get state. */
icetGetIntegerv(ICET_RANK, &rank);
icetGetIntegerv(ICET_NUM_PROCESSES, &num_proc);
icetGetIntegerv(ICET_NUM_TILES, &num_tiles);
icetGetIntegerv(ICET_TILE_MAX_PIXELS, &max_pixels);
display_nodes = icetUnsafeStateGet(ICET_DISPLAY_NODES);
tile_viewports = icetUnsafeStateGet(ICET_TILE_VIEWPORTS);
icetGetIntegerv(ICET_TILE_DISPLAYED, &tile_displayed);
/* Allocate buffers. */
icetResizeBuffer( icetFullImageSize(max_pixels)
+ icetSparseImageSize(max_pixels)*2
+ sizeof(struct node_info)*num_proc
+ sizeof(GLboolean)*num_proc*num_tiles);
imageBuffer = icetReserveBufferMem(icetFullImageSize(max_pixels));
inImage = icetReserveBufferMem(icetSparseImageSize(max_pixels));
outImage = icetReserveBufferMem(icetSparseImageSize(max_pixels));
info = icetReserveBufferMem(sizeof(struct node_info)*num_proc);
all_contained_tmasks
= icetReserveBufferMem(sizeof(GLboolean)*num_proc*num_tiles);
icetGetBooleanv(ICET_ALL_CONTAINED_TILES_MASKS, all_contained_tmasks);
/* Initialize info array. */
for (node = 0; node < num_proc; node++) {
info[node].rank = node;
info[node].tile_held = -1; /* Id of tile image held in memory. */
info[node].num_contained = 0; /* # of images to be rendered. */
for (tile = 0; tile < num_tiles; tile++) {
if (all_contained_tmasks[node*num_tiles + tile]) {
info[node].num_contained++;
}
}
}
#define CONTAINS_TILE(nodei, tile) \
(all_contained_tmasks[info[nodei].rank*num_tiles+(tile)])
tile_held = -1;
do {
int recv_node;
tiles_transfered = 0;
sort_by_contained(info, num_proc);
for (node = 0; node < num_proc; node++) {
info[node].tile_sending = -1;
info[node].tile_receiving = -1;
}
for (recv_node = 0; recv_node < num_proc; recv_node++) {
struct node_info *recv_info = info + recv_node;
if (recv_info->tile_receiving >= 0) continue;
if (recv_info->tile_held >= 0) {
/* This node is holding a tile. It must either send or
receive this tile. */
if (find_sender(info, num_proc, recv_node, recv_info->tile_held,
display_nodes[recv_info->tile_held],
num_tiles, all_contained_tmasks)) {
tiles_transfered = 1;
continue;
}
/* Could not find a match for a sender, how about someone who
can receive it? */
if ( (recv_info->tile_sending < 0)
&& (recv_info->rank != display_nodes[recv_info->tile_held])
&& find_receiver(info, num_proc, recv_node,
recv_info->tile_held,
display_nodes[recv_info->tile_held],
num_tiles, all_contained_tmasks) ) {
tiles_transfered = 1;
} else {
/* Could not send or receive. Give up. */
continue;
}
}
/* OK. Let's try to receive any tile that we still have. */
for (tile = 0; tile < num_tiles; tile++) {
if ( ( !CONTAINS_TILE(recv_node, tile)
&& (display_nodes[tile] != recv_info->rank) )
|| (recv_info->tile_sending == tile) ) continue;
if (find_sender(info, num_proc, recv_node, tile,
display_nodes[tile], num_tiles,
all_contained_tmasks)) {
tiles_transfered = 1;
break;
}
}
}
/* Now that we figured out who is sending to who, do the actual
send and receive. */
my_info = NULL;
for (node = 0; node < num_proc; node++) {
if (info[node].rank == rank) {
my_info = info + node;
break;
}
}
do_send_receive(my_info, tile_held, max_pixels, num_tiles,
tile_viewports, all_contained_tmasks,
imageBuffer, inImage, outImage);
tile_held = my_info->tile_held;
} while (tiles_transfered);
/* It's possible that a composited image ended up on a processor that */
/* is not the display node for that image. Do one last round of */
/* transfers to make sure all the tiles ended up in the right place. */
for (node = 0; node < num_proc; node++) {
if (info[node].rank == rank) {
my_info = info + node;
break;
}
}
my_info->tile_receiving = -1;
my_info->tile_sending = -1;
if ((my_info->tile_held >= 0) && (my_info->tile_held != tile_displayed)) {
/* I'm holding an image that does not belong to me. Ship it off. */
my_info->tile_sending = my_info->tile_held;
my_info->send_dest = display_nodes[my_info->tile_held];
my_info->tile_held = -1;
}
if ((my_info->tile_held != tile_displayed) && (tile_displayed >= 0)) {
/* Someone may be holding an image that belongs to me. Check. */
for (node = 0; node < num_proc; node++) {
if (info[node].tile_held == tile_displayed) {
my_info->tile_receiving = tile_displayed;
my_info->recv_src = info[node].rank;
my_info->tile_held = tile_displayed;
break;
}
}
}
do_send_receive(my_info, tile_held,
max_pixels, num_tiles, tile_viewports, all_contained_tmasks,
imageBuffer, inImage, outImage);
tile_held = my_info->tile_held;
/* Hacks for when "this" tile was not rendered. */
if ((tile_displayed >= 0) && (tile_displayed != tile_held)) {
if (all_contained_tmasks[rank*num_tiles + tile_displayed]) {
/* Only "this" node draws "this" tile. Because the image never */
/* needed to be transferred, it was never rendered above. Just */
/* render it now. */
icetRaiseDebug("Rendering tile to display.");
/* This may uncessarily read a buffer if not outputing an input
buffer */
icetGetTileImage(tile_displayed, imageBuffer);
} else {
/* "This" tile is blank. */
icetRaiseDebug("Returning blank image.");
icetInitializeImage(imageBuffer, max_pixels);
icetClearImage(imageBuffer);
}
}
return imageBuffer;
}
static void icetDoSendRecvLarge(const IceTInt *sendIds,
const IceTBoolean *myDestMask,
const IceTBoolean *mySrcMask,
IceTBoolean messagesInOrder,
IceTGenerateData generateDataFunc,
IceTHandleData handleDataFunc,
IceTVoid *incomingBuffer,
IceTSizeType bufferSize)
{
IceTInt comm_size;
IceTInt rank;
const IceTInt *composite_order;
const IceTInt *process_orders;
IceTInt order_rank;
IceTInt recv_order_idx;
IceTInt send_order_idx;
enum IceTCommIndices { RECV_IDX = 0, SEND_IDX = 1 };
IceTCommRequest requests[2];
enum IceTIterState recv_iter_state;
enum IceTIterState send_iter_state;
icetGetIntegerv(ICET_NUM_PROCESSES, &comm_size);
icetGetIntegerv(ICET_RANK, &rank);
if (messagesInOrder) {
composite_order = icetUnsafeStateGetInteger(ICET_COMPOSITE_ORDER);
process_orders = icetUnsafeStateGetInteger(ICET_PROCESS_ORDERS);
} else {
composite_order = NULL;
process_orders = NULL;
}
/* We'll just handle send to self as a special case. */
if (myDestMask[rank]) {
IceTSizeType data_size;
IceTVoid *data;
icetRaiseDebug("Sending to self.");
data = (*generateDataFunc)(sendIds[rank], rank, &data_size);
(*handleDataFunc)(data, rank);
}
/* We have to create a communication pattern that is guaranteed not to
deadlock even if we don't know what everyone is sending. To do this, we
use a particular type of communication pattern. If order does not
matter, then each process asynchronously sends to rank+1 and receives
from rank-1 (with wraparound between 0 and num_proc-1). Because there
every sender has receive and vice versa, this is guaranteed by MPI to
complete. This is then repeated sending to rank+2, rank+3, and so on.
The ordered communication is similar except that there is no wraparound.
If the sender or receiver is outside the range, no communication happens.
The communication is then repeated in the other direction. Again, all
send/receives are matched and eventually everyone sends/receives to/from
everyone.
The trick we are going to pull is realize that the actual messages are
probably sparse. Thus, we won't actually do a send/receive if no message
is being passed. However, this does not change the fact that all sends
and receives are matched up. */
if (messagesInOrder) {
order_rank = process_orders[rank];
} else {
order_rank = rank;
}
recv_order_idx = send_order_idx = order_rank;
recv_iter_state = ICET_SEND_RECV_LARGE_ITER_BACKWARD;
send_iter_state = ICET_SEND_RECV_LARGE_ITER_FORWARD;
requests[0] = requests[1] = ICET_COMM_REQUEST_NULL;
while (ICET_TRUE) {
icetSendRecvLargePostReceive(mySrcMask,
messagesInOrder,
order_rank,
incomingBuffer,
bufferSize,
&recv_order_idx,
&recv_iter_state,
&requests[RECV_IDX]);
icetSendRecvLargePostSend(sendIds,
myDestMask,
messagesInOrder,
generateDataFunc,
order_rank,
&send_order_idx,
&send_iter_state,
&requests[SEND_IDX]);
/* If finished with all messages, quit. */ {
IceTBoolean finished;
finished = (send_iter_state == ICET_SEND_RECV_LARGE_ITER_DONE);
finished &= (recv_iter_state == ICET_SEND_RECV_LARGE_ITER_DONE);
if (finished) break;
}
/* Wait for some some message to come in before continuing. */ {
int request_finished_idx = icetCommWaitany(2, requests);
if (request_finished_idx == RECV_IDX) {
IceTInt src_rank;
if (messagesInOrder) {
src_rank = composite_order[recv_order_idx];
} else {
src_rank = recv_iter_state;
}
(*handleDataFunc)(incomingBuffer, src_rank);
}
}
}
}
static IceTImage splitStrategy(void)
{
int *tile_groups;
int my_tile;
int group_size;
int fragment_size;
GLint rank;
GLint num_proc;
GLint num_tiles;
GLint max_pixels;
GLint *tile_contribs;
GLint total_image_count;
GLint *display_nodes;
GLint tile_displayed;
GLenum output_buffers;
GLint num_contained_tiles;
GLint *contained_tiles_list;
GLboolean *all_contained_tiles_masks;
int tile, image, node;
int num_allocated;
IceTSparseImage *incoming;
IceTSparseImage outgoing;
IceTImage imageFragment;
IceTImage fullImage;
int num_requests;
IceTCommRequest *requests;
int first_incoming = 1;
icetRaiseDebug("In splitStrategy");
icetGetIntegerv(ICET_RANK, &rank);
icetGetIntegerv(ICET_NUM_PROCESSES, &num_proc);
icetGetIntegerv(ICET_NUM_TILES, &num_tiles);
icetGetIntegerv(ICET_TILE_MAX_PIXELS, &max_pixels);
tile_contribs = icetUnsafeStateGet(ICET_TILE_CONTRIB_COUNTS);
icetGetIntegerv(ICET_TOTAL_IMAGE_COUNT, &total_image_count);
display_nodes = icetUnsafeStateGet(ICET_DISPLAY_NODES);
icetGetIntegerv(ICET_TILE_DISPLAYED, &tile_displayed);
icetGetIntegerv(ICET_NUM_CONTAINED_TILES, &num_contained_tiles);
contained_tiles_list = icetUnsafeStateGet(ICET_CONTAINED_TILES_LIST);
all_contained_tiles_masks
= icetUnsafeStateGet(ICET_ALL_CONTAINED_TILES_MASKS);
/* Special case: no images rendered whatsoever. */
if (total_image_count < 1) {
icetRaiseDebug("Not rendering any images. Quit early.");
if (tile_displayed >= 0) {
icetResizeBuffer(icetFullImageSize(max_pixels));
fullImage = icetReserveBufferMem(icetFullImageSize(max_pixels));
icetInitializeImage(fullImage, max_pixels);
icetClearImage(fullImage);
} else {
fullImage = NULL;
}
return fullImage;
}
tile_groups = malloc(sizeof(int)*(num_tiles+1));
num_allocated = 0;
tile_groups[0] = 0;
/* Set entry of tile_groups[i+1] to the number of processes to help
compose the image in tile i. */
for (tile = 0; tile < num_tiles; tile++) {
int allocate = (tile_contribs[tile]*num_proc)/total_image_count;
if ((allocate < 1) && (tile_contribs[tile] > 0)) {
allocate = 1;
}
tile_groups[tile+1] = allocate;
num_allocated += allocate;
}
/* Make the number of processes allocated equal exactly the number of
processes available. */
while (num_allocated < num_proc) {
/* Add processes to the tile with the lowest process:image ratio. */
int min_id = -1;
float min_ratio = (float)num_proc;
for (tile = 0; tile < num_tiles; tile++) {
float ratio;
/* Don't even consider tiles with no contributors. */
if (tile_contribs[tile] == 0) continue;
ratio = (float)tile_groups[tile+1]/tile_contribs[tile];
if (ratio < min_ratio) {
min_ratio = ratio;
min_id = tile;
}
}
#ifdef DEBUG
if (min_id < 0) {
icetRaiseError("Could not find candidate to add tile.",
ICET_SANITY_CHECK_FAIL);
}
#endif
tile_groups[min_id+1]++;
num_allocated++;
}
while (num_allocated > num_proc) {
/* Remove processes from the tile with the highest process:image
ratio. */
int max_id = -1;
float max_ratio = 0;
for (tile = 0; tile < num_tiles; tile++) {
float ratio;
/* Don't even consider tiles with a minimum allocation. */
if (tile_groups[tile+1] <= 1) continue;
ratio = (float)tile_groups[tile+1]/tile_contribs[tile];
if (ratio > max_ratio) {
max_ratio = ratio;
max_id = tile;
}
}
#ifdef DEBUG
if (max_id < 0) {
icetRaiseError("Could not find candidate to remove tile.",
ICET_SANITY_CHECK_FAIL);
}
#endif
tile_groups[max_id+1]--;
num_allocated--;
}
/* Processes are assigned sequentially from 0 to N to each tile as
needed. Change each tile_groups[i] entry to be the lowest rank of the
processes assigned to tile i. Thus the processes assigned to tile i
are tile_groups[i] through tile_groups[i+1]-1. */
for (tile = 1; tile < num_tiles; tile++) {
tile_groups[tile] += tile_groups[tile-1];
}
tile_groups[num_tiles] = num_proc;
/* Figure out which tile I am assigned to. */
for (my_tile = 0; rank >= tile_groups[my_tile+1]; my_tile++);
group_size = tile_groups[my_tile+1] - tile_groups[my_tile];
fragment_size = max_pixels/group_size;
num_requests = tile_contribs[my_tile];
if (num_requests < 2) num_requests = 2;
icetResizeBuffer( sizeof(IceTSparseImage)*tile_contribs[my_tile]
+ icetFullImageSize(fragment_size)
+ icetSparseImageSize(max_pixels)
+ icetFullImageSize(max_pixels)
+ icetSparseImageSize(fragment_size)*tile_contribs[my_tile]
+ sizeof(IceTCommRequest)*num_requests);
incoming
= icetReserveBufferMem(sizeof(IceTSparseImage)*tile_contribs[my_tile]);
outgoing = icetReserveBufferMem(icetSparseImageSize(max_pixels));
imageFragment = icetReserveBufferMem(icetFullImageSize(fragment_size));
fullImage = icetReserveBufferMem(icetFullImageSize(max_pixels));
requests = icetReserveBufferMem(sizeof(IceTCommRequest)*num_requests);
/* Set up asynchronous receives for all incoming image fragments. */
/* for (image = 0; image < tile_contribs[my_tile]; image++) { */
/* incoming[image] */
/* = icetReserveBufferMem(icetSparseImageSize(fragment_size)); */
/* MPI_Irecv(incoming[image], icetSparseImageSize(fragment_size), */
/* MPI_BYTE, MPI_ANY_SOURCE, IMAGE_DATA, */
/* icetGetCommunicator(), requests + image); */
/* } */
for (image = 0, node = 0; image < tile_contribs[my_tile]; node++) {
if (all_contained_tiles_masks[node*num_tiles + my_tile]) {
icetRaiseDebug1("Setting up receive from node %d", node);
incoming[image]
= icetReserveBufferMem(icetSparseImageSize(fragment_size));
requests[image] =
ICET_COMM_IRECV(incoming[image],
icetSparseImageSize(fragment_size),
ICET_BYTE, node, IMAGE_DATA);
image++;
}
}
/* Render and send all tile images I am rendering. */
for (image = 0; image < num_contained_tiles; image++) {
int sending_frag_size;
int compressedSize;
GLuint offset;
tile = contained_tiles_list[image];
icetGetTileImage(tile, fullImage);
icetRaiseDebug1("Got image for tile %d", tile);
offset = 0;
sending_frag_size = max_pixels/(tile_groups[tile+1]-tile_groups[tile]);
for (node = tile_groups[tile]; node < tile_groups[tile+1]; node++) {
icetRaiseDebug2("Sending tile %d to node %d", tile, node);
compressedSize = icetCompressSubImage(fullImage, offset,
sending_frag_size, outgoing);
icetAddSentBytes(compressedSize);
ICET_COMM_SEND(outgoing, compressedSize,
ICET_BYTE, node, IMAGE_DATA);
offset += sending_frag_size;
}
}
/* Wait for images to come in and Z compare them. */
for (image = 0; image < tile_contribs[my_tile]; image++) {
int idx;
idx = ICET_COMM_WAITANY(tile_contribs[my_tile], requests);
if (first_incoming) {
icetRaiseDebug1("Got first image (%d).", idx);
icetDecompressImage(incoming[idx], imageFragment);
first_incoming = 0;
} else {
icetRaiseDebug1("Got subsequent image (%d).", idx);
icetCompressedComposite(imageFragment,
incoming[idx], 1);
}
}
/* Send composited fragment to display process. */
icetGetIntegerv(ICET_OUTPUT_BUFFERS, (GLint *)&output_buffers);
if ((output_buffers & ICET_COLOR_BUFFER_BIT) != 0) {
icetAddSentBytes(4*fragment_size);
requests[0] = ICET_COMM_ISEND(icetGetImageColorBuffer(imageFragment),
4*fragment_size,
ICET_BYTE, display_nodes[my_tile],
COLOR_DATA);
}
if ((output_buffers & ICET_DEPTH_BUFFER_BIT) != 0) {
icetAddSentBytes(4*fragment_size);
requests[1] = ICET_COMM_ISEND(icetGetImageDepthBuffer(imageFragment),
fragment_size, ICET_INT,
display_nodes[my_tile], DEPTH_DATA);
}
/* If I am displaying a tile, receive image data. */
if (tile_displayed >= 0) {
icetInitializeImage(fullImage, max_pixels);
/* Check to make sure tile is not blank. */
if (tile_groups[tile_displayed+1] > tile_groups[tile_displayed]) {
int my_frag_size = max_pixels/( tile_groups[tile_displayed+1]
- tile_groups[tile_displayed]);
if ((output_buffers & ICET_COLOR_BUFFER_BIT) != 0) {
GLubyte *cb = icetGetImageColorBuffer(fullImage);
for (node = tile_groups[tile_displayed];
node < tile_groups[tile_displayed+1]; node++) {
icetRaiseDebug1("Getting final color fragment from %d",
node);
ICET_COMM_RECV(cb, 4*my_frag_size,
ICET_BYTE, node, COLOR_DATA);
cb += 4*my_frag_size;
}
}
if ((output_buffers & ICET_DEPTH_BUFFER_BIT) != 0) {
GLuint *db = icetGetImageDepthBuffer(fullImage);
for (node = tile_groups[tile_displayed];
node < tile_groups[tile_displayed+1]; node++) {
icetRaiseDebug1("Getting final depth fragment from %d",
node);
ICET_COMM_RECV(db, my_frag_size,
ICET_INT, node, DEPTH_DATA);
db += my_frag_size;
}
}
} else {
icetClearImage(fullImage);
}
}
if ((output_buffers & ICET_COLOR_BUFFER_BIT) != 0) {
ICET_COMM_WAIT(requests);
}
if ((output_buffers & ICET_DEPTH_BUFFER_BIT) != 0) {
ICET_COMM_WAIT(requests + 1);
}
free(tile_groups);
return fullImage;
}