static IceTImage vtreeCompose(void)
{
GLint rank, num_proc;
GLint num_tiles;
GLint max_pixels;
GLint *display_nodes;
GLint tile_displayed;
GLboolean *all_contained_tmasks;
GLint *tile_viewports;
IceTImage imageBuffer;
IceTSparseImage inImage, outImage;
struct node_info *info;
struct node_info *my_info;
int tile, node;
int tiles_transfered;
int tile_held = -1;
icetRaiseDebug("In vtreeCompose");
/* Get state. */
icetGetIntegerv(ICET_RANK, &rank);
icetGetIntegerv(ICET_NUM_PROCESSES, &num_proc);
icetGetIntegerv(ICET_NUM_TILES, &num_tiles);
icetGetIntegerv(ICET_TILE_MAX_PIXELS, &max_pixels);
display_nodes = icetUnsafeStateGet(ICET_DISPLAY_NODES);
tile_viewports = icetUnsafeStateGet(ICET_TILE_VIEWPORTS);
icetGetIntegerv(ICET_TILE_DISPLAYED, &tile_displayed);
/* Allocate buffers. */
icetResizeBuffer( icetFullImageSize(max_pixels)
+ icetSparseImageSize(max_pixels)*2
+ sizeof(struct node_info)*num_proc
+ sizeof(GLboolean)*num_proc*num_tiles);
imageBuffer = icetReserveBufferMem(icetFullImageSize(max_pixels));
inImage = icetReserveBufferMem(icetSparseImageSize(max_pixels));
outImage = icetReserveBufferMem(icetSparseImageSize(max_pixels));
info = icetReserveBufferMem(sizeof(struct node_info)*num_proc);
all_contained_tmasks
= icetReserveBufferMem(sizeof(GLboolean)*num_proc*num_tiles);
icetGetBooleanv(ICET_ALL_CONTAINED_TILES_MASKS, all_contained_tmasks);
/* Initialize info array. */
for (node = 0; node < num_proc; node++) {
info[node].rank = node;
info[node].tile_held = -1; /* Id of tile image held in memory. */
info[node].num_contained = 0; /* # of images to be rendered. */
for (tile = 0; tile < num_tiles; tile++) {
if (all_contained_tmasks[node*num_tiles + tile]) {
info[node].num_contained++;
}
}
}
#define CONTAINS_TILE(nodei, tile) \
(all_contained_tmasks[info[nodei].rank*num_tiles+(tile)])
tile_held = -1;
do {
int recv_node;
tiles_transfered = 0;
sort_by_contained(info, num_proc);
for (node = 0; node < num_proc; node++) {
info[node].tile_sending = -1;
info[node].tile_receiving = -1;
}
for (recv_node = 0; recv_node < num_proc; recv_node++) {
struct node_info *recv_info = info + recv_node;
if (recv_info->tile_receiving >= 0) continue;
if (recv_info->tile_held >= 0) {
/* This node is holding a tile. It must either send or
receive this tile. */
if (find_sender(info, num_proc, recv_node, recv_info->tile_held,
display_nodes[recv_info->tile_held],
num_tiles, all_contained_tmasks)) {
tiles_transfered = 1;
continue;
}
/* Could not find a match for a sender, how about someone who
can receive it? */
if ( (recv_info->tile_sending < 0)
&& (recv_info->rank != display_nodes[recv_info->tile_held])
&& find_receiver(info, num_proc, recv_node,
recv_info->tile_held,
display_nodes[recv_info->tile_held],
num_tiles, all_contained_tmasks) ) {
tiles_transfered = 1;
} else {
/* Could not send or receive. Give up. */
continue;
}
}
/* OK. Let's try to receive any tile that we still have. */
for (tile = 0; tile < num_tiles; tile++) {
if ( ( !CONTAINS_TILE(recv_node, tile)
&& (display_nodes[tile] != recv_info->rank) )
|| (recv_info->tile_sending == tile) ) continue;
if (find_sender(info, num_proc, recv_node, tile,
display_nodes[tile], num_tiles,
all_contained_tmasks)) {
tiles_transfered = 1;
break;
}
}
}
/* Now that we figured out who is sending to who, do the actual
send and receive. */
my_info = NULL;
for (node = 0; node < num_proc; node++) {
if (info[node].rank == rank) {
my_info = info + node;
break;
}
}
do_send_receive(my_info, tile_held, max_pixels, num_tiles,
tile_viewports, all_contained_tmasks,
imageBuffer, inImage, outImage);
tile_held = my_info->tile_held;
} while (tiles_transfered);
/* It's possible that a composited image ended up on a processor that */
/* is not the display node for that image. Do one last round of */
/* transfers to make sure all the tiles ended up in the right place. */
for (node = 0; node < num_proc; node++) {
if (info[node].rank == rank) {
my_info = info + node;
break;
}
}
my_info->tile_receiving = -1;
my_info->tile_sending = -1;
if ((my_info->tile_held >= 0) && (my_info->tile_held != tile_displayed)) {
/* I'm holding an image that does not belong to me. Ship it off. */
my_info->tile_sending = my_info->tile_held;
my_info->send_dest = display_nodes[my_info->tile_held];
my_info->tile_held = -1;
}
if ((my_info->tile_held != tile_displayed) && (tile_displayed >= 0)) {
/* Someone may be holding an image that belongs to me. Check. */
for (node = 0; node < num_proc; node++) {
if (info[node].tile_held == tile_displayed) {
my_info->tile_receiving = tile_displayed;
my_info->recv_src = info[node].rank;
my_info->tile_held = tile_displayed;
break;
}
}
}
do_send_receive(my_info, tile_held,
max_pixels, num_tiles, tile_viewports, all_contained_tmasks,
imageBuffer, inImage, outImage);
tile_held = my_info->tile_held;
/* Hacks for when "this" tile was not rendered. */
if ((tile_displayed >= 0) && (tile_displayed != tile_held)) {
if (all_contained_tmasks[rank*num_tiles + tile_displayed]) {
/* Only "this" node draws "this" tile. Because the image never */
/* needed to be transferred, it was never rendered above. Just */
/* render it now. */
icetRaiseDebug("Rendering tile to display.");
/* This may uncessarily read a buffer if not outputing an input
buffer */
icetGetTileImage(tile_displayed, imageBuffer);
} else {
/* "This" tile is blank. */
icetRaiseDebug("Returning blank image.");
icetInitializeImage(imageBuffer, max_pixels);
icetClearImage(imageBuffer);
}
}
return imageBuffer;
}
static IceTImage splitStrategy(void)
{
int *tile_groups;
int my_tile;
int group_size;
int fragment_size;
GLint rank;
GLint num_proc;
GLint num_tiles;
GLint max_pixels;
GLint *tile_contribs;
GLint total_image_count;
GLint *display_nodes;
GLint tile_displayed;
GLenum output_buffers;
GLint num_contained_tiles;
GLint *contained_tiles_list;
GLboolean *all_contained_tiles_masks;
int tile, image, node;
int num_allocated;
IceTSparseImage *incoming;
IceTSparseImage outgoing;
IceTImage imageFragment;
IceTImage fullImage;
int num_requests;
IceTCommRequest *requests;
int first_incoming = 1;
icetRaiseDebug("In splitStrategy");
icetGetIntegerv(ICET_RANK, &rank);
icetGetIntegerv(ICET_NUM_PROCESSES, &num_proc);
icetGetIntegerv(ICET_NUM_TILES, &num_tiles);
icetGetIntegerv(ICET_TILE_MAX_PIXELS, &max_pixels);
tile_contribs = icetUnsafeStateGet(ICET_TILE_CONTRIB_COUNTS);
icetGetIntegerv(ICET_TOTAL_IMAGE_COUNT, &total_image_count);
display_nodes = icetUnsafeStateGet(ICET_DISPLAY_NODES);
icetGetIntegerv(ICET_TILE_DISPLAYED, &tile_displayed);
icetGetIntegerv(ICET_NUM_CONTAINED_TILES, &num_contained_tiles);
contained_tiles_list = icetUnsafeStateGet(ICET_CONTAINED_TILES_LIST);
all_contained_tiles_masks
= icetUnsafeStateGet(ICET_ALL_CONTAINED_TILES_MASKS);
/* Special case: no images rendered whatsoever. */
if (total_image_count < 1) {
icetRaiseDebug("Not rendering any images. Quit early.");
if (tile_displayed >= 0) {
icetResizeBuffer(icetFullImageSize(max_pixels));
fullImage = icetReserveBufferMem(icetFullImageSize(max_pixels));
icetInitializeImage(fullImage, max_pixels);
icetClearImage(fullImage);
} else {
fullImage = NULL;
}
return fullImage;
}
tile_groups = malloc(sizeof(int)*(num_tiles+1));
num_allocated = 0;
tile_groups[0] = 0;
/* Set entry of tile_groups[i+1] to the number of processes to help
compose the image in tile i. */
for (tile = 0; tile < num_tiles; tile++) {
int allocate = (tile_contribs[tile]*num_proc)/total_image_count;
if ((allocate < 1) && (tile_contribs[tile] > 0)) {
allocate = 1;
}
tile_groups[tile+1] = allocate;
num_allocated += allocate;
}
/* Make the number of processes allocated equal exactly the number of
processes available. */
while (num_allocated < num_proc) {
/* Add processes to the tile with the lowest process:image ratio. */
int min_id = -1;
float min_ratio = (float)num_proc;
for (tile = 0; tile < num_tiles; tile++) {
float ratio;
/* Don't even consider tiles with no contributors. */
if (tile_contribs[tile] == 0) continue;
ratio = (float)tile_groups[tile+1]/tile_contribs[tile];
if (ratio < min_ratio) {
min_ratio = ratio;
min_id = tile;
}
}
#ifdef DEBUG
if (min_id < 0) {
icetRaiseError("Could not find candidate to add tile.",
ICET_SANITY_CHECK_FAIL);
}
#endif
tile_groups[min_id+1]++;
num_allocated++;
}
while (num_allocated > num_proc) {
/* Remove processes from the tile with the highest process:image
ratio. */
int max_id = -1;
float max_ratio = 0;
for (tile = 0; tile < num_tiles; tile++) {
float ratio;
/* Don't even consider tiles with a minimum allocation. */
if (tile_groups[tile+1] <= 1) continue;
ratio = (float)tile_groups[tile+1]/tile_contribs[tile];
if (ratio > max_ratio) {
max_ratio = ratio;
max_id = tile;
}
}
#ifdef DEBUG
if (max_id < 0) {
icetRaiseError("Could not find candidate to remove tile.",
ICET_SANITY_CHECK_FAIL);
}
#endif
tile_groups[max_id+1]--;
num_allocated--;
}
/* Processes are assigned sequentially from 0 to N to each tile as
needed. Change each tile_groups[i] entry to be the lowest rank of the
processes assigned to tile i. Thus the processes assigned to tile i
are tile_groups[i] through tile_groups[i+1]-1. */
for (tile = 1; tile < num_tiles; tile++) {
tile_groups[tile] += tile_groups[tile-1];
}
tile_groups[num_tiles] = num_proc;
/* Figure out which tile I am assigned to. */
for (my_tile = 0; rank >= tile_groups[my_tile+1]; my_tile++);
group_size = tile_groups[my_tile+1] - tile_groups[my_tile];
fragment_size = max_pixels/group_size;
num_requests = tile_contribs[my_tile];
if (num_requests < 2) num_requests = 2;
icetResizeBuffer( sizeof(IceTSparseImage)*tile_contribs[my_tile]
+ icetFullImageSize(fragment_size)
+ icetSparseImageSize(max_pixels)
+ icetFullImageSize(max_pixels)
+ icetSparseImageSize(fragment_size)*tile_contribs[my_tile]
+ sizeof(IceTCommRequest)*num_requests);
incoming
= icetReserveBufferMem(sizeof(IceTSparseImage)*tile_contribs[my_tile]);
outgoing = icetReserveBufferMem(icetSparseImageSize(max_pixels));
imageFragment = icetReserveBufferMem(icetFullImageSize(fragment_size));
fullImage = icetReserveBufferMem(icetFullImageSize(max_pixels));
requests = icetReserveBufferMem(sizeof(IceTCommRequest)*num_requests);
/* Set up asynchronous receives for all incoming image fragments. */
/* for (image = 0; image < tile_contribs[my_tile]; image++) { */
/* incoming[image] */
/* = icetReserveBufferMem(icetSparseImageSize(fragment_size)); */
/* MPI_Irecv(incoming[image], icetSparseImageSize(fragment_size), */
/* MPI_BYTE, MPI_ANY_SOURCE, IMAGE_DATA, */
/* icetGetCommunicator(), requests + image); */
/* } */
for (image = 0, node = 0; image < tile_contribs[my_tile]; node++) {
if (all_contained_tiles_masks[node*num_tiles + my_tile]) {
icetRaiseDebug1("Setting up receive from node %d", node);
incoming[image]
= icetReserveBufferMem(icetSparseImageSize(fragment_size));
requests[image] =
ICET_COMM_IRECV(incoming[image],
icetSparseImageSize(fragment_size),
ICET_BYTE, node, IMAGE_DATA);
image++;
}
}
/* Render and send all tile images I am rendering. */
for (image = 0; image < num_contained_tiles; image++) {
int sending_frag_size;
int compressedSize;
GLuint offset;
tile = contained_tiles_list[image];
icetGetTileImage(tile, fullImage);
icetRaiseDebug1("Got image for tile %d", tile);
offset = 0;
sending_frag_size = max_pixels/(tile_groups[tile+1]-tile_groups[tile]);
for (node = tile_groups[tile]; node < tile_groups[tile+1]; node++) {
icetRaiseDebug2("Sending tile %d to node %d", tile, node);
compressedSize = icetCompressSubImage(fullImage, offset,
sending_frag_size, outgoing);
icetAddSentBytes(compressedSize);
ICET_COMM_SEND(outgoing, compressedSize,
ICET_BYTE, node, IMAGE_DATA);
offset += sending_frag_size;
}
}
/* Wait for images to come in and Z compare them. */
for (image = 0; image < tile_contribs[my_tile]; image++) {
int idx;
idx = ICET_COMM_WAITANY(tile_contribs[my_tile], requests);
if (first_incoming) {
icetRaiseDebug1("Got first image (%d).", idx);
icetDecompressImage(incoming[idx], imageFragment);
first_incoming = 0;
} else {
icetRaiseDebug1("Got subsequent image (%d).", idx);
icetCompressedComposite(imageFragment,
incoming[idx], 1);
}
}
/* Send composited fragment to display process. */
icetGetIntegerv(ICET_OUTPUT_BUFFERS, (GLint *)&output_buffers);
if ((output_buffers & ICET_COLOR_BUFFER_BIT) != 0) {
icetAddSentBytes(4*fragment_size);
requests[0] = ICET_COMM_ISEND(icetGetImageColorBuffer(imageFragment),
4*fragment_size,
ICET_BYTE, display_nodes[my_tile],
COLOR_DATA);
}
if ((output_buffers & ICET_DEPTH_BUFFER_BIT) != 0) {
icetAddSentBytes(4*fragment_size);
requests[1] = ICET_COMM_ISEND(icetGetImageDepthBuffer(imageFragment),
fragment_size, ICET_INT,
display_nodes[my_tile], DEPTH_DATA);
}
/* If I am displaying a tile, receive image data. */
if (tile_displayed >= 0) {
icetInitializeImage(fullImage, max_pixels);
/* Check to make sure tile is not blank. */
if (tile_groups[tile_displayed+1] > tile_groups[tile_displayed]) {
int my_frag_size = max_pixels/( tile_groups[tile_displayed+1]
- tile_groups[tile_displayed]);
if ((output_buffers & ICET_COLOR_BUFFER_BIT) != 0) {
GLubyte *cb = icetGetImageColorBuffer(fullImage);
for (node = tile_groups[tile_displayed];
node < tile_groups[tile_displayed+1]; node++) {
icetRaiseDebug1("Getting final color fragment from %d",
node);
ICET_COMM_RECV(cb, 4*my_frag_size,
ICET_BYTE, node, COLOR_DATA);
cb += 4*my_frag_size;
}
}
if ((output_buffers & ICET_DEPTH_BUFFER_BIT) != 0) {
GLuint *db = icetGetImageDepthBuffer(fullImage);
for (node = tile_groups[tile_displayed];
node < tile_groups[tile_displayed+1]; node++) {
icetRaiseDebug1("Getting final depth fragment from %d",
node);
ICET_COMM_RECV(db, my_frag_size,
ICET_INT, node, DEPTH_DATA);
db += my_frag_size;
}
}
} else {
icetClearImage(fullImage);
}
}
if ((output_buffers & ICET_COLOR_BUFFER_BIT) != 0) {
ICET_COMM_WAIT(requests);
}
if ((output_buffers & ICET_DEPTH_BUFFER_BIT) != 0) {
ICET_COMM_WAIT(requests + 1);
}
free(tile_groups);
return fullImage;
}
