/* This function may work on any distribution of the bipartite graph */ int Zoltan_ZG_Register(ZZ* zz, ZG* graph, int* properties) { static char *yo = "Zoltan_ZG_Register"; int ierr = ZOLTAN_OK; int *props; struct Zoltan_DD_Struct *dd; int size; ZOLTAN_ID_PTR GID; ZOLTAN_TRACE_ENTER(zz, yo); size = graph->mtx.mtx.nY; dd = graph->mtx.mtx.ddY; if (graph->bipartite) { /* Need to construct another properties array with only the fixed elements ! */ int vertlno; if (graph->fixObj) { dd = graph->mtx.mtx.ddX; } props = (int*)ZOLTAN_MALLOC(sizeof(int)*size); if (graph->mtx.mtx.nY && props == NULL) MEMORY_ERROR; GID = ZOLTAN_MALLOC_GID_ARRAY(zz, size); if (size && GID == NULL) MEMORY_ERROR; for (size = 0, vertlno = 0 ; vertlno < graph->mtx.mtx.nY ; ++vertlno) { if (graph->fixed_vertices[vertlno]) { props[size] = properties[vertlno]; ZOLTAN_SET_GID(zz, GID+ size*zz->Num_GID, graph->mtx.mtx.yGID+vertlno*zz->Num_GID); size ++; } } } else { props = properties; GID = graph->mtx.mtx.yGID; if (graph->mtx.mtx.ddY == NULL) { ierr = Zoltan_DD_Create (&graph->mtx.mtx.ddY, zz->Communicator, 1, zz->Num_GID, 1, graph->mtx.mtx.globalY/zz->Num_Proc, 0); CHECK_IERR; /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(graph->mtx.mtx.ddY, graph->mtx.mtx.globalX/zz->Num_Proc); } dd = graph->mtx.mtx.ddY; } /* Make our new numbering public */ ierr = Zoltan_DD_Update (dd, GID, NULL, NULL, props, size); CHECK_IERR; End: if (graph->bipartite) { ZOLTAN_FREE(&props); ZOLTAN_FREE(&GID); } ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_Matrix_Complete(ZZ* zz,Zoltan_matrix* m) { static char *yo = "Zoltan_Matrix_Complete"; int ierr = ZOLTAN_OK; if(m->completed) return (ZOLTAN_OK); ZOLTAN_TRACE_ENTER(zz, yo); if (m->yend != m->ystart +1 ) {/* Not in compact mode yet */ int y; /* I have to also rewrites all the pinarrays */ for (y = 1 ; y <= m->nY ; ++y) { int length; if (m->ystart[y] == m->yend[y-1]) /* No hole */ continue; length = m->yend[y]-m->ystart[y]; memcpy(m->pinGNO+m->yend[y-1], m->pinGNO+m->ystart[y], length*sizeof(ZOLTAN_GNO_TYPE)); memcpy(m->pinwgt+m->yend[y-1]*m->pinwgtdim, m->pinGNO+m->ystart[y]*m->pinwgtdim, length*sizeof(float)*m->pinwgtdim); m->ystart[y] = m->yend[y-1]; m->yend[y] = m->ystart[y] + length; } ZOLTAN_FREE(&m->yend); m->yend = m->ystart + 1; } /* Update data directories */ m->yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, m->nY); m->ypid = (int*) ZOLTAN_MALLOC(m->nY * sizeof(int)); if (m->bipartite) m->ybipart = (int*) ZOLTAN_MALLOC(m->nY * sizeof(int)); if (m->nY && ((m->yGID == NULL) || (m->ypid == NULL) || (m->bipartite && m->ybipart == NULL))) MEMORY_ERROR; /* Get Informations about Y */ Zoltan_DD_Find (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, m->yGID, (char *)m->ypid, m->ybipart, m->nY, NULL); if (m->ddY != m->ddX) { Zoltan_DD_Destroy(&m->ddY); m->ddY = NULL; } m->completed = 1; End: ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
/* Return an array of locally owned GID */ ZOLTAN_ID_PTR Zoltan_Matrix_Get_GID(ZZ* zz, Zoltan_matrix* m) { ZOLTAN_ID_PTR yGID; yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, m->nY); if (m->nY && yGID == NULL) return (NULL); /* Get Informations about Y */ Zoltan_DD_Find (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, yGID, NULL, NULL, m->nY, NULL); return (yGID); }
static int actual_arrays( ZZ *zz, int num_gid_entries, int num_lid_entries, int num, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids, int *procs, int *to_part, int *actual_num, ZOLTAN_ID_PTR *actual_gids, ZOLTAN_ID_PTR *actual_lids, int **actual_procs, int **actual_to_part, int *actual_allocated ) { char *yo = "actual_arrays"; int i, j; /* * Test whether to pack objects that have changed partition * but not changed processor. * If packing them, the actual objects == objects passed to this function. * If not packing them, build arrays with them stripped out. */ *actual_allocated = 0; if (!(zz->Migrate.Only_Proc_Changes)) { /* Pack all objects, even if they are not changing processor. */ *actual_num = num; *actual_gids = gids; *actual_lids = lids; *actual_procs = procs; *actual_to_part = to_part; } else { /* zz->Migrate.Only_Proc_Changes */ /* Pack only objects that are actually changing processor. */ *actual_num = 0; for (i = 0; i < num; i++) if (procs[i] != zz->Proc) (*actual_num)++; if (*actual_num == num) { /* Number of actual objects == number of objects in input arrays. */ /* No stripping needed. */ *actual_gids = gids; *actual_lids = lids; *actual_procs = procs; *actual_to_part = to_part; } else if (*actual_num != num && *actual_num > 0) { /* Number of actual_num < num. Build arrays */ /* containing only actual objects. */ *actual_allocated = 1; *actual_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, *actual_num); *actual_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, *actual_num); *actual_procs = (int *) ZOLTAN_MALLOC(sizeof(int) * (*actual_num)); if (to_part != NULL) *actual_to_part = (int *) ZOLTAN_MALLOC(sizeof(int)*(*actual_num)); if (*actual_gids == NULL || (num_lid_entries && *actual_lids == NULL) || *actual_procs == NULL || (to_part != NULL && *actual_to_part == NULL)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory Error."); Zoltan_Multifree(__FILE__, __LINE__, 4, actual_gids, actual_lids, actual_procs, actual_to_part); return (ZOLTAN_MEMERR); } for (j = 0, i = 0; i < num; i++) { if (procs[i] != zz->Proc) { ZOLTAN_SET_GID(zz, *actual_gids + j*num_gid_entries, gids + i*num_gid_entries); if (num_lid_entries) ZOLTAN_SET_LID(zz, *actual_lids + j*num_lid_entries, lids + i*num_lid_entries); (*actual_procs)[j] = procs[i]; if (to_part) (*actual_to_part)[j] = to_part[i]; j++; } } } } return ZOLTAN_OK; }
int Zoltan_Migrate( ZZ *zz, /* Zoltan structure. */ int num_import, /* Number of non-local objects assigned to the processor in the new decomposition. */ ZOLTAN_ID_PTR import_global_ids, /* Array of global IDs for non-local objects assigned to this processor in the new decomposition; this field can be NULL if the application doesn't provide import IDs.*/ ZOLTAN_ID_PTR import_local_ids, /* Array of local IDs for non-local objects assigned to the processor in the new decomposition; this field can be NULL if the application does not provide import IDs. */ int *import_procs, /* Array of processor IDs of processors owning the non-local objects that are assigned to this processor in the new decomposition; this field can be NULL if the application does not provide import IDs. */ int *import_to_part, /* Array of partition numbers to which imported objects should be assigned. */ int num_export, /* Number of objs to be exported to other processors to establish the new decomposition. */ ZOLTAN_ID_PTR export_global_ids, /* Array of global IDs of objects to be exported to other processors to establish the new decomposition. */ ZOLTAN_ID_PTR export_local_ids, /* Array of local IDs of objects to be exported to other processors to establish the new decomposition. */ int *export_procs, /* Array of processor IDs to which objects will be exported to establish the new decomposition. */ int *export_to_part /* Array of partition numbers to which exported objects should be assigned. */ ) { /* * Routine to help perform migration. If migration pre-processing routine * (ZOLTAN_PRE_MIGRATE_FN) is specified, this routine first calls that fn. * It then calls a function to obtain the size of the migrating objects * (ZOLTAN_OBJ_SIZE_FN). The routine next calls an application-specified * object packing routine (ZOLTAN_PACK_OBJ_FN) for each object * to be exported. It develops the needed communication map to move the * objects to other processors. It performs the communication according * to the map, and then calls an application-specified object unpacking * routine (ZOLTAN_UNPACK_OBJ_FN) for each object imported. */ char *yo = "Zoltan_Migrate"; int num_gid_entries, num_lid_entries; /* lengths of global & local ids */ int *sizes = NULL; /* sizes (in bytes) of the object data for export. */ int id_size; /* size (in bytes) of ZOLTAN_GID + padding for alignment */ int tag_size; /* size (in bytes) of ZOLTAN_GID + one int (for message size) */ char *export_buf = NULL; /* buffer for packing export data. */ char *import_buf = NULL; /* buffer for receiving imported data. */ char *tmp; /* temporary pointer into buffers. */ int i; /* loop counter. */ int tmp_size; /* size of a single object's data. */ int *idx = NULL; /* index used for multi-fn packs and unpacks. */ int idx_cnt = 0; /* index counter for idx array. */ ZOLTAN_ID_PTR tmp_id = NULL; /* pointer to storage for a global ID in comm buf */ ZOLTAN_ID_PTR lid; /* temporary pointer to a local ID; used to pass NULL to query functions when NUM_LID_ENTRIES=0. */ ZOLTAN_COMM_OBJ *imp_plan = NULL; /* Comm obj built from import lists. */ ZOLTAN_COMM_OBJ *exp_plan = NULL; /* Comm obj built from export lists. */ int msgtag, msgtag2; /* Tags for communication routines */ int total_send_size; /* Total size of outcoming message (in #items) */ int total_recv_size; /* Total size of incoming message (in #items) */ int aligned_int; /* size of an int padded for alignment */ int dest; /* temporary destination partition. */ int include_parts = 0; /* flag indicating whether partition info is provided */ int ierr = ZOLTAN_OK; int actual_num_exp = 0; int actual_exp_allocated = 0; ZOLTAN_ID_PTR actual_exp_gids = NULL; /* Arrays containing only objs to */ ZOLTAN_ID_PTR actual_exp_lids = NULL; /* actually be packed. Objs that */ int *actual_exp_procs = NULL; /* are changing partition but not */ int *actual_exp_to_part = NULL; /* processor may not be included. */ int actual_num_imp = 0; int actual_imp_allocated = 0; ZOLTAN_ID_PTR actual_imp_gids = NULL; /* Arrays containing only objs to */ ZOLTAN_ID_PTR actual_imp_lids = NULL; /* actually be imported. Objs that */ int *actual_imp_procs = NULL; /* are changing partition but not */ int *actual_imp_to_part = NULL; /* processor may not be included. */ ZOLTAN_TRACE_ENTER(zz, yo); /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { goto End; } /* * Check that all procs use the same id types. */ ierr = check_input(zz, ((num_export >= 0 && export_to_part) || (num_import >= 0 && import_to_part)), &include_parts); if (ierr != ZOLTAN_OK) goto End; num_gid_entries = zz->Num_GID; num_lid_entries = zz->Num_LID; /* * Check that all necessary query functions are available. */ if (zz->Get_Obj_Size == NULL && zz->Get_Obj_Size_Multi == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a " "ZOLTAN_OBJ_SIZE_FN or ZOLTAN_OBJ_SIZE_MULTI_FN function " "to use the migration-help tools."); ierr = ZOLTAN_FATAL; goto End; } if (zz->Pack_Obj == NULL && zz->Pack_Obj_Multi == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a " "ZOLTAN_PACK_OBJ_FN or ZOLTAN_PACK_OBJ_MULTI_FN function " "to use the migration-help tools."); ierr = ZOLTAN_FATAL; goto End; } if (zz->Unpack_Obj == NULL && zz->Unpack_Obj_Multi == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a " "ZOLTAN_UNPACK_OBJ_FN or ZOLTAN_UNPACK_OBJ_MULTI_FN function " "to use the migration-help tools."); ierr = ZOLTAN_FATAL; goto End; } if (num_export >= 0) { /* Build the actual export arrays */ ierr = actual_arrays(zz, num_gid_entries, num_lid_entries, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &actual_num_exp, &actual_exp_gids, &actual_exp_lids, &actual_exp_procs, &actual_exp_to_part, &actual_exp_allocated); if (ierr < 0) goto End; /* Compute communication map based on actual exports. */ msgtag = 32767; ierr = Zoltan_Comm_Create(&exp_plan, actual_num_exp, actual_exp_procs, zz->Communicator, msgtag, &actual_num_imp); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create."); goto End; } } else if (num_import >= 0) { /* Build the actual import arrays */ ierr = actual_arrays(zz, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, &actual_num_imp, &actual_imp_gids, &actual_imp_lids, &actual_imp_procs, &actual_imp_to_part, &actual_imp_allocated); if (ierr < 0) goto End; /* Compute communication map based on imports. */ msgtag = 32767; ierr = Zoltan_Comm_Create(&imp_plan, actual_num_imp, actual_imp_procs, zz->Communicator, msgtag, &actual_num_exp); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create."); goto End; } /* Compute actual export lists for packing objects */ if (actual_num_exp > 0) { actual_exp_allocated = 1; actual_exp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_exp); actual_exp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, actual_num_exp); actual_exp_procs = (int *) ZOLTAN_MALLOC(sizeof(int) * actual_num_exp); if (include_parts) actual_exp_to_part = (int *) ZOLTAN_MALLOC(sizeof(int)*actual_num_exp); if (actual_exp_gids == NULL || (num_lid_entries && actual_exp_lids == NULL) || actual_exp_procs == NULL || (import_to_part != NULL && actual_exp_to_part == NULL)) { Zoltan_Multifree(__FILE__, __LINE__, 4, &actual_exp_gids, &actual_exp_lids, &actual_exp_procs, &actual_exp_to_part); ierr = ZOLTAN_MEMERR; goto End; } } msgtag2 = 32766; ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_gids, (int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)), (char *) actual_exp_gids); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } if (num_lid_entries) { msgtag2--; ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_lids, (int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries), (char *) actual_exp_lids); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } } Zoltan_Comm_Info(imp_plan, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, actual_exp_procs, NULL); if (include_parts) { msgtag2--; ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_to_part, (int) sizeof(int), (char *) actual_exp_to_part); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } } /* Create inverse plan (i.e., plan based on exports) so can set * variable sizes. * (Zoltan_Comm_Do_Reverse(imp_plan, ...) allows sending variable * but does not tell how large to allocate receive buffer. */ ierr = Zoltan_Comm_Invert_Plan(&imp_plan); exp_plan = imp_plan; imp_plan = NULL; } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Import or export lists needed."); ierr = ZOLTAN_FATAL; goto End; } if (zz->Migrate.Pre_Migrate_PP != NULL) { zz->Migrate.Pre_Migrate_PP(zz->Migrate.Pre_Migrate_PP_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PRE_MIGRATE_PP_FN function."); goto End; } } if (zz->Migrate.Pre_Migrate != NULL) { zz->Migrate.Pre_Migrate(zz->Migrate.Pre_Migrate_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, num_export, export_global_ids, export_local_ids, export_procs, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PRE_MIGRATE_FN function."); goto End; } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done pre-migration processing"); id_size = Zoltan_Align(num_gid_entries * sizeof(ZOLTAN_ID_TYPE)); /* Note that alignment is not strictly necessary when ZOLTAN_ID_TYPE is int or unsigned int. */ aligned_int = Zoltan_Align(sizeof(int)); tag_size = id_size + aligned_int; /* * For each object, allow space for its global ID and its data plus * one int (for the object data size). * Zoltan will pack the global IDs; the application must pack the data * through the pack routine. Zoltan needs the global IDs for unpacking, * as the order of the data received during communication is not * necessarily the same order as import_global_ids[]. * Zoltan also needs to communicate the sizes of the objects because * only the sender knows the size of each object. */ if (actual_num_exp > 0) { sizes = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int)); if (!sizes) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } if (zz->Get_Obj_Size_Multi != NULL) { zz->Get_Obj_Size_Multi(zz->Get_Obj_Size_Multi_Data, num_gid_entries, num_lid_entries, actual_num_exp, actual_exp_gids, actual_exp_lids, sizes, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_OBJ_SIZE_MULTI function."); goto End; } } else { for (i = 0; i < actual_num_exp; i++) { lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL); sizes[i] = zz->Get_Obj_Size(zz->Get_Obj_Size_Data, num_gid_entries, num_lid_entries, &(actual_exp_gids[i*num_gid_entries]), lid, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_OBJ_SIZE function."); goto End; } } } total_send_size = 0; for (i = 0; i < actual_num_exp; i++) { sizes[i] = Zoltan_Align(sizes[i]); total_send_size += sizes[i] + tag_size; } export_buf = (char *) ZOLTAN_CALLOC(total_send_size, sizeof(char)); if (!export_buf) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } if (zz->Pack_Obj_Multi != NULL) { /* Allocate an index array for ZOLTAN_PACK_OBJ_MULTI_FN. */ idx = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int)); if (!idx) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } } /* * Pack the objects for export. */ idx_cnt = 0; tmp = export_buf; for (i = 0; i < actual_num_exp; i++) { /* Pack the object's global ID */ tmp_id = (ZOLTAN_ID_PTR) tmp; ZOLTAN_SET_GID(zz, tmp_id, &(actual_exp_gids[i*num_gid_entries])); tmp += id_size; /* Pack the object's size */ *((int *)tmp) = sizes[i]; tmp += aligned_int; /* If using ZOLTAN_PACK_OBJ_MULTI_FN, build the index array. */ idx_cnt += tag_size; if (idx != NULL) { idx[i] = idx_cnt; } tmp += sizes[i]; idx_cnt += sizes[i]; } if (zz->Pack_Obj_Multi != NULL) { if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Packing objects with multi-pack\n", zz->Proc, yo); } zz->Pack_Obj_Multi(zz->Pack_Obj_Multi_Data, num_gid_entries, num_lid_entries, actual_num_exp, actual_exp_gids, actual_exp_lids, (actual_exp_to_part!=NULL ? actual_exp_to_part : actual_exp_procs), sizes, idx, export_buf, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PACK_OBJ_MULTI function."); goto End; } } else { tmp = export_buf + tag_size; for (i = 0; i < actual_num_exp; i++) { if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Packing object with gid ", zz->Proc, yo); ZOLTAN_PRINT_GID(zz, &(actual_exp_gids[i*num_gid_entries])); printf("size = %d bytes\n", sizes[i]); } /* Pack the object's data */ lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL); dest = (actual_exp_to_part != NULL ? actual_exp_to_part[i] : actual_exp_procs[i]); zz->Pack_Obj(zz->Pack_Obj_Data, num_gid_entries, num_lid_entries, &(actual_exp_gids[i*num_gid_entries]), lid, dest, sizes[i], tmp, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PACK_OBJ function."); goto End; } tmp += sizes[i] + tag_size; } } ZOLTAN_FREE(&idx); tmp_id = NULL; } ZOLTAN_TRACE_DETAIL(zz, yo, "Done packing objects"); /* Modify sizes[] to contain message sizes, not object sizes */ for (i=0; i<actual_num_exp; i++) { sizes[i] += tag_size; } msgtag--; ierr = Zoltan_Comm_Resize(exp_plan, sizes, msgtag, &total_recv_size); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Resize."); goto End; } if (actual_num_imp > 0) { import_buf = (char *) ZOLTAN_MALLOC(total_recv_size); if (!import_buf) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } } /* * Send the export data using the communication plan. */ msgtag2 = 32765; ierr = Zoltan_Comm_Do(exp_plan, msgtag2, export_buf, 1, import_buf); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } /* * Free whatever memory we can. */ Zoltan_Comm_Destroy(&exp_plan); ZOLTAN_FREE(&export_buf); ZOLTAN_FREE(&sizes); ZOLTAN_TRACE_DETAIL(zz, yo, "Done communication"); /* * Perform application-specified processing before unpacking the data. */ if (zz->Migrate.Mid_Migrate_PP != NULL) { zz->Migrate.Mid_Migrate_PP(zz->Migrate.Mid_Migrate_PP_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_MID_MIGRATE_PP_FN function."); goto End; } } if (zz->Migrate.Mid_Migrate != NULL) { zz->Migrate.Mid_Migrate(zz->Migrate.Mid_Migrate_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, num_export, export_global_ids, export_local_ids, export_procs, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_MID_MIGRATE_FN function."); goto End; } } /* * Unpack the object data. */ if (actual_num_imp > 0) { if (zz->Unpack_Obj_Multi != NULL) { /* Allocate and fill input arrays for Unpack_Obj_Multi. */ sizes = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int)); tmp_id = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_imp); idx = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int)); if (!sizes || !tmp_id || !idx) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } tmp = import_buf; idx_cnt = 0; for (i = 0; i < actual_num_imp; i++) { /* Unpack the object's global ID */ ZOLTAN_SET_GID(zz, &(tmp_id[i*num_gid_entries]), (ZOLTAN_ID_PTR) tmp); tmp += id_size; /* Unpack the object's size */ sizes[i] = *((int *)tmp); tmp += aligned_int; /* If using ZOLTAN_UNPACK_OBJ_MULTI_FN, build the index array. */ idx_cnt += tag_size; if (idx != NULL) { idx[i] = idx_cnt; } tmp += sizes[i]; idx_cnt += sizes[i]; } if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Unpacking objects with multi-fn\n", zz->Proc,yo); } zz->Unpack_Obj_Multi(zz->Unpack_Obj_Multi_Data, num_gid_entries, actual_num_imp, tmp_id, sizes, idx, import_buf, &ierr); ZOLTAN_FREE(&import_buf); ZOLTAN_FREE(&sizes); ZOLTAN_FREE(&tmp_id); ZOLTAN_FREE(&idx); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_UNPACK_OBJ_MULTI_FN."); goto End; } } else { tmp = import_buf; for (i = 0; i < actual_num_imp; i++) { tmp_size = *((int *)(tmp + id_size)); if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Unpacking object with gid ", zz->Proc, yo); ZOLTAN_PRINT_GID(zz, (ZOLTAN_ID_PTR)tmp); printf("size = %d bytes\n", tmp_size); } /* Unpack the object's data */ zz->Unpack_Obj(zz->Unpack_Obj_Data, num_gid_entries, (ZOLTAN_ID_PTR) tmp, tmp_size, tmp + tag_size, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_UNPACK_OBJ_FN."); goto End; } tmp += (tmp_size + tag_size); } ZOLTAN_FREE(&import_buf); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done unpacking objects"); if (zz->Migrate.Post_Migrate_PP != NULL) { zz->Migrate.Post_Migrate_PP(zz->Migrate.Post_Migrate_PP_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_POST_MIGRATE_PP_FN function."); goto End; } } if (zz->Migrate.Post_Migrate != NULL) { zz->Migrate.Post_Migrate(zz->Migrate.Post_Migrate_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, num_export, export_global_ids, export_local_ids, export_procs, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_POST_MIGRATE_FN function."); goto End; } } End: if (actual_exp_allocated) { Zoltan_Multifree(__FILE__, __LINE__, 4, &actual_exp_gids, &actual_exp_lids, &actual_exp_procs, &actual_exp_to_part); } if (actual_imp_allocated) { Zoltan_Multifree(__FILE__, __LINE__, 4, &actual_imp_gids, &actual_imp_lids, &actual_imp_procs, &actual_imp_to_part); } if (ierr < 0) { if (exp_plan) Zoltan_Comm_Destroy(&exp_plan); Zoltan_Multifree(__FILE__, __LINE__, 5, &import_buf, &tmp_id, &sizes, &idx, &export_buf); } ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
/* * void malloc_new_objects(); * * gets the tags being imported into this processor, and sets up the * import_tags array, and the nrectags array. */ static int malloc_new_objects(ZZ *zz, int nsentags, pRegion exported_tags, ZOLTAN_ID_PTR exported_gids, ZOLTAN_ID_PTR exported_lids, int *tag_pids, int *nstags, pRegion *ex_tags, pRegion prev_tags, ZOLTAN_ID_PTR prev_gids, ZOLTAN_ID_PTR prev_lids, int npimtags, float *c3) { char *yo = "malloc_new_objects"; int i; /* index counter */ int nreceives; /* number of messages received */ pRegion t_b_exp; /* array of tags to be exported */ pRegion tmp = NULL; ZOLTAN_ID_PTR tmp_gids = NULL; ZOLTAN_ID_PTR tmp_lids = NULL; int msgtag, msgtag2; int j; int ierr = ZOLTAN_OK; int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; float im_load; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */ im_load = 0; msgtag = 32767; ierr = Zoltan_Comm_Create(&comm_plan, nsentags, tag_pids, zz->Communicator, msgtag, &nreceives); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Create."); ZOLTAN_TRACE_EXIT(zz, yo); return(ierr); } if (nreceives > 0) { tmp = (pRegion) ZOLTAN_MALLOC(nreceives * sizeof(Region)); tmp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, nreceives); tmp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, nreceives); if(tmp == NULL || !tmp_gids || (num_lid_entries && !tmp_lids)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } } msgtag2 = 32766; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_tags, sizeof(Region), (char *) tmp); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return(ierr); } msgtag2--; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_gids, sizeof(ZOLTAN_ID_TYPE)*num_gid_entries, (char *) tmp_gids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); fprintf(stderr, "OCT %s Error %s returned from Zoltan_Comm_Do\n", yo, (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return(ierr); } if (num_lid_entries > 0) { msgtag2--; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_lids, sizeof(ZOLTAN_ID_TYPE)*num_lid_entries, (char *) tmp_lids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return(ierr); } } ierr = Zoltan_Comm_Destroy(&comm_plan); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Destroy."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&tmp); return(ierr); } /* get each message sent, and store region in import array */ j=0; for (i=0; i<nreceives; i++) { im_load += tmp[i].Weight; if(tmp[i].newProc != zz->Proc) { j++; } } if((j + npimtags) != 0) { /* malloc import array */ if((t_b_exp = (pRegion)ZOLTAN_MALLOC((j+npimtags)*sizeof(Region)))==NULL){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return ZOLTAN_MEMERR; } } else t_b_exp = NULL; /* setup return pointer */ (*ex_tags) = t_b_exp; j=0; for (i=0; i<nreceives; i++) { if(tmp[i].newProc != zz->Proc) { t_b_exp[j] = tmp[i]; t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz); t_b_exp[j].Local_ID = ZOLTAN_MALLOC_LID(zz); if (!(t_b_exp[j].Global_ID) || (num_lid_entries && !(t_b_exp[j].Local_ID))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return ZOLTAN_MEMERR; } ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID, &(tmp_gids[i*num_gid_entries])); ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID, &(tmp_lids[i*num_lid_entries])); j++; } } if(npimtags > 0) { for(i=0; i<npimtags; i++) { t_b_exp[j] = prev_tags[i]; t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz); t_b_exp[j].Local_ID = ZOLTAN_MALLOC_LID(zz); if (!(t_b_exp[j].Global_ID) || (num_lid_entries && !(t_b_exp[j].Local_ID))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return ZOLTAN_MEMERR; } ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID, &(prev_gids[i*num_gid_entries])); ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID, &(prev_lids[i*num_lid_entries])); j++; } } *nstags = j; ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); if((*nstags == 0) && (*ex_tags != NULL)) { ZOLTAN_TRACE_DETAIL(zz, yo, "Fatal error, import tags not empty but no tags received\n"); return ZOLTAN_FATAL; } *c3 = im_load; return ierr; }
/* * void tag_regions() * Iterates through the list of octants on the processor and finds which * are to be migrated. It then looks at the region list for those octants * and stores the migrating regions into the export_tags array. */ static int tag_regions(ZZ *zz, pOctant *octs, int *newpids, int nocts, Region **exported_tags, ZOLTAN_ID_PTR *exported_gids, ZOLTAN_ID_PTR *exported_lids, int *nsentags, int **tag_pids, Region **p_tags, ZOLTAN_ID_PTR *p_gids, ZOLTAN_ID_PTR *p_lids, int *npimtags, float *c2, int *max_objs) { char *yo = "tag_regions"; int i; /* index counter */ pRegion regionlist; /* list of region on this processor */ int index; /* index counter */ int index2; /* yet another index counter */ int count; /* count of objects exported form this processor */ int count2; /* count of objects that are kept on processor */ int count3; int *exported_pids; /* array of pids where regions are being exported to */ pRegion mtags; /* object tags of objects to be migrated */ pRegion ptags; /* tags of objects that were previously migrated */ float ex_load; int ierr = ZOLTAN_OK; int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; ex_load = 0; (*max_objs) = 0; if (!nsentags) return ierr; /* find how many objects have been exported */ count = 0; /* find number of local objs to export */ count2 = 0; count3 = 0; for (i=0; i<nocts; i++) { if(Zoltan_Oct_isTerminal(octs[i])) { (*max_objs) += Zoltan_Oct_nRegions(octs[i]); regionlist = Zoltan_Oct_regionlist(octs[i]); while(regionlist != NULL) { count3++; if(regionlist->Proc != zz->Proc) { count++; if(newpids[i] != zz->Proc) regionlist->newProc = newpids[i]; else regionlist->newProc = zz->Proc; } else { if(newpids[i] != zz->Proc) { count2++; regionlist->newProc = newpids[i]; } else regionlist->newProc = zz->Proc; } regionlist = regionlist->next; /* get next region */ } } } #if 0 { { if (newpids[i]!=zz->Proc) { count+=Zoltan_Oct_nRegions(octs[i]); } else { pRegion regions; regions = Zoltan_Oct_regionlist(octs[i]); while(regions != NULL) { if(regions->Proc != zz->Proc) count2++; regions = regions->next; } } } } #endif /* set up the return pointers */ *nsentags = count; *npimtags = count2; if (!exported_tags) { return ierr; } if (count > 0) { /* allocate some space */ if((mtags=(pRegion)ZOLTAN_MALLOC((unsigned)count*sizeof(Region)))==NULL){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if((exported_pids = (int *)ZOLTAN_MALLOC((unsigned)count*sizeof(int))) == NULL){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&mtags); return ZOLTAN_MEMERR; } *exported_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, count); *exported_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, count); if(!(*exported_gids) || (num_lid_entries && !(*exported_lids))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&mtags); ZOLTAN_FREE(&exported_pids); return ZOLTAN_MEMERR; } } else { mtags = NULL; exported_pids = NULL; *exported_gids = NULL; *exported_lids = NULL; } /* set up return pointers */ *exported_tags=mtags; *tag_pids = exported_pids; if (count2 > 0) { /* allocate some space */ if((ptags=(pRegion)ZOLTAN_MALLOC((unsigned)count2*sizeof(Region)))==NULL){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient Memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&mtags); ZOLTAN_FREE(&exported_pids); ZOLTAN_FREE(exported_gids); ZOLTAN_FREE(exported_lids); return ZOLTAN_MEMERR; } *p_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, count2); *p_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, count2); if(!(*p_gids) || (num_lid_entries && !(*p_lids))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient Memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&mtags); ZOLTAN_FREE(&exported_pids); ZOLTAN_FREE(exported_gids); ZOLTAN_FREE(exported_lids); ZOLTAN_FREE(&ptags); ZOLTAN_FREE(p_gids); ZOLTAN_FREE(p_lids); return ZOLTAN_MEMERR; } } else { ptags = NULL; *p_gids = NULL; *p_lids = NULL; } /* set up return pointers */ *p_tags=ptags; index = index2 = 0; for (i=0; i<nocts; i++) { if(Zoltan_Oct_isTerminal(octs[i])) { regionlist = Zoltan_Oct_regionlist(octs[i]); while(regionlist != NULL) { if(regionlist->Proc != zz->Proc) { /* place information in the appropritate array */ mtags[index] = *regionlist; ZOLTAN_SET_GID(zz, &((*exported_gids)[index*num_gid_entries]), regionlist->Global_ID); ZOLTAN_SET_LID(zz, &((*exported_lids)[index*num_lid_entries]), regionlist->Local_ID); /*ex_load += (float)(regionlist->Weight);*/ exported_pids[index] = regionlist->Proc; index++; /* increment counter */ } else if(newpids[i] != zz->Proc) { ptags[index2] = *regionlist; /* get region information */ ZOLTAN_SET_GID(zz, &((*p_gids)[index2*num_gid_entries]), regionlist->Global_ID); ZOLTAN_SET_LID(zz, &((*p_lids)[index2*num_lid_entries]), regionlist->Local_ID); index2++; /* increment counter */ } regionlist = regionlist->next; /* get next region */ } } } if (index!=count) { /* error check */ ZOLTAN_TRACE_DETAIL(zz, yo, "Fatal error, inconsistent number of regions.\n"); return ZOLTAN_FATAL; } *c2 = ex_load; return ierr; }
static int _Zoltan_Get_Obj_List( ZZ *zz, int *num_obj, ZOLTAN_ID_PTR *global_ids, ZOLTAN_ID_PTR *local_ids, int wdim, float **objwgts, int **parts, int special_malloc ) { char *yo = "_Zoltan_Get_Obj_List"; int i, n; int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; int alloced_gids = 0, alloced_lids = 0; int gid_off, lid_off; ZOLTAN_ID_PTR lid, next_lid; /* Temporary pointers to local IDs; used to pass NULL to query functions when NUM_LID_ENTRIES == 0. */ float *next_objwgt; /* Temporarry pointer to an object weight; used to pass NULL to query functions when wdim=0 */ int ierr = ZOLTAN_OK; ZOLTAN_TRACE_ENTER(zz, yo); *num_obj = 0; *objwgts = NULL; if (zz->Get_Num_Obj != NULL) { *num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); goto End; } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); ierr = ZOLTAN_FATAL; goto End; } if (*num_obj > 0) { /* * Test global_ids and local_ids for NULL. * Should be NULL for doing partitioning. * Should not be NULL if doing ordering. */ if (special_malloc){ if (*global_ids == NULL) { Zoltan_Special_Malloc(zz, (void **)global_ids, *num_obj, ZOLTAN_SPECIAL_MALLOC_GID); alloced_gids = 1; } if (*local_ids == NULL) { Zoltan_Special_Malloc(zz, (void **)local_ids, *num_obj, ZOLTAN_SPECIAL_MALLOC_LID); alloced_lids = 1; } Zoltan_Special_Malloc(zz, (void **)parts, *num_obj, ZOLTAN_SPECIAL_MALLOC_INT); } else{ if (*global_ids == NULL) { *global_ids = ZOLTAN_MALLOC_GID_ARRAY(zz, *num_obj); alloced_gids = 1; } if (*local_ids == NULL) { *local_ids = ZOLTAN_MALLOC_LID_ARRAY(zz, *num_obj); alloced_lids = 1; } *parts = (int *) ZOLTAN_MALLOC(*num_obj * sizeof(int)); } if (wdim > 0) *objwgts = (float*) ZOLTAN_MALLOC (sizeof(float) * *num_obj * wdim); if ((*global_ids == NULL) || (num_lid_entries > 0 && *local_ids == NULL) || (*parts == NULL) || (wdim > 0 && *objwgts == NULL)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory Error."); ierr = ZOLTAN_MEMERR; goto End; } if (zz->Get_Obj_List != NULL){ /* Get object list directly */ zz->Get_Obj_List(zz->Get_Obj_List_Data, num_gid_entries, num_lid_entries, *global_ids, *local_ids, wdim, *objwgts, &ierr); } else if ((zz->Get_First_Obj != NULL) && (zz->Get_Next_Obj != NULL)){ /* Use iterator functions to loop through object list */ if (zz->Get_First_Obj(zz->Get_First_Obj_Data, num_gid_entries, num_lid_entries, *global_ids, *local_ids, wdim, *objwgts, &ierr)){ n = *num_obj; i = 0; while (!ierr && (i<n-1)){ gid_off = i * num_gid_entries; lid_off = i * num_lid_entries; lid = (num_lid_entries ? &((*local_ids)[lid_off]) : NULL); next_lid = (num_lid_entries ? &((*local_ids)[lid_off+num_lid_entries]) : NULL); next_objwgt = (wdim ? (*objwgts) + (i+1)*wdim : NULL); zz->Get_Next_Obj(zz->Get_Next_Obj_Data, num_gid_entries, num_lid_entries, &((*global_ids)[gid_off]), lid, &((*global_ids)[gid_off+num_gid_entries]), next_lid, wdim, next_objwgt, &ierr); i++; } } } else { /* No way to get objects */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_OBJ_LIST_FN or " "ZOLTAN_FIRST_OBJ_FN/ZOLTAN_NEXT_OBJ_FN."); ierr = ZOLTAN_FATAL; goto End; } /* Get partition information for objects. */ /* Call user-callback if provided; otherwise, all parts == zz->Proc */ if (zz->Get_Part == NULL && zz->Get_Part_Multi == NULL) { for (i = 0; i < *num_obj; i++) (*parts)[i] = zz->Proc; } else if (zz->Get_Part_Multi) { zz->Get_Part_Multi(zz->Get_Part_Multi_Data, num_gid_entries, num_lid_entries, *num_obj, *global_ids, *local_ids, *parts, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from ZOLTAN_PART_MULTI_FN"); goto End; } } else { for (i = 0; i < *num_obj; i++) { lid = (num_lid_entries ? &((*local_ids)[i*num_lid_entries]) : NULL); (*parts)[i] = zz->Get_Part(zz->Get_Part_Data, num_gid_entries, num_lid_entries, &((*global_ids)[i*num_gid_entries]), lid, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from ZOLTAN_PART_FN"); goto End; } } } } End: if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error found; no lists returned."); if (special_malloc){ if (alloced_gids) Zoltan_Special_Free(zz, (void **)global_ids, ZOLTAN_SPECIAL_MALLOC_GID); if (alloced_lids) Zoltan_Special_Free(zz, (void **)local_ids, ZOLTAN_SPECIAL_MALLOC_LID); Zoltan_Special_Free(zz, (void **)parts, ZOLTAN_SPECIAL_MALLOC_INT); } else{ if (alloced_gids) ZOLTAN_FREE(global_ids); if (alloced_lids) ZOLTAN_FREE(local_ids); ZOLTAN_FREE(parts); } ZOLTAN_FREE(objwgts); } ZOLTAN_TRACE_EXIT(zz, yo); return ierr; }
/* Performs a permutation of the matrix, perm_y A perm_y^t. * TODO: at this time we only do symmetric permutations (don't know xGNO !). */ int Zoltan_Matrix_Permute(ZZ* zz, Zoltan_matrix *m, int* perm_y) { static char *yo = "Zoltan_Matrix_Permute"; int ierr = ZOLTAN_OK; int *pinGNO = NULL; ZOLTAN_ID_PTR yGID=NULL; float *ywgt=NULL; struct Zoltan_DD_Struct *dd; ZOLTAN_TRACE_ENTER(zz, yo); /* First apply y permutation */ if (m->completed) { /* We directly know the good arrays */ yGID = m->yGID; ywgt = m->ywgt; if (m->ddY == NULL || m->ddY != m->ddX) { /* We have to create again the DD */ /* We have to define ddY : yGNO, yGID, ywgt */ ierr = Zoltan_DD_Create (&m->ddY, zz->Communicator, 1, zz->Num_GID, m->ywgtdim*sizeof(float)/sizeof(int), m->globalY/zz->Num_Proc, 0); /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(m->ddY, m->globalY/zz->Num_Proc); } } else { /* We have to get these fields */ /* Update data directories */ yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, m->nY); ywgt = (float*) ZOLTAN_MALLOC(m->nY * sizeof(float) * m->ywgtdim); if (m->nY && (yGID == NULL || (m->ywgtdim && ywgt == NULL))) MEMORY_ERROR; /* Get Informations about Y */ Zoltan_DD_Find (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, yGID, (ZOLTAN_ID_PTR)ywgt, NULL, m->nY, NULL); } memcpy (m->yGNO, perm_y, m->nY*sizeof(int)); /* Get Informations about Y */ Zoltan_DD_Update (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, yGID, (ZOLTAN_ID_PTR)ywgt, NULL, m->nY); ZOLTAN_FREE (&yGID); ZOLTAN_FREE (&ywgt); /* We have to define dd : old_yGNO, new_yGNO */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, 1, 1, 0, m->globalY/zz->Num_Proc, 0); /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, m->globalY/zz->Num_Proc); Zoltan_DD_Update (dd, (ZOLTAN_ID_PTR)m->yGNO, (ZOLTAN_ID_PTR)perm_y, NULL, NULL, m->nY); pinGNO = (int*)ZOLTAN_MALLOC(m->nPins*sizeof(int)); if (m->nPins && pinGNO == NULL) MEMORY_ERROR; Zoltan_DD_Find (dd, (ZOLTAN_ID_PTR)m->pinGNO, (ZOLTAN_ID_PTR)pinGNO, NULL, NULL, m->nY, NULL); ZOLTAN_FREE(&m->pinGNO); m->pinGNO = pinGNO; pinGNO = NULL; End: ZOLTAN_FREE (&pinGNO); ZOLTAN_FREE (&yGID); ZOLTAN_FREE (&ywgt); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_Order_Get_GID_Order( struct Zoltan_Struct *zz, ZOLTAN_ID_PTR global_ids, ZOLTAN_ID_PTR order_ids ) { int * proctab; ZOLTAN_ID_PTR sendgidtab; int * sendtab, *recvtab; int i; int nrecv; struct Zoltan_Comm_Obj *plan; int ierr = ZOLTAN_OK; int *vtxdist; int tag = 24061986; int offset; zz->Order.gidrank = order_ids; if (!strcmp(zz->Order.order_type, "LOCAL")) { for (i = 0 ; i < zz->Order.nbr_objects ; ++i) { order_ids[i] = global_ids[zz->Order.rank[i] - zz->Order.start_index]; } return (ZOLTAN_OK); } ierr = Zoltan_Get_Distribution(zz, &vtxdist); if (ierr != ZOLTAN_OK) return (ierr); proctab = (int*) ZOLTAN_MALLOC(3*zz->Order.nbr_objects*sizeof(int)); sendgidtab = ZOLTAN_MALLOC_GID_ARRAY(zz, 2*zz->Order.nbr_objects); if (proctab == NULL) { ZOLTAN_FREE(&vtxdist); return (ZOLTAN_MEMERR); } if (sendgidtab == NULL) { ZOLTAN_FREE(&proctab); ZOLTAN_FREE(&vtxdist); return (ZOLTAN_MEMERR); } sendtab = proctab + zz->Order.nbr_objects; recvtab = sendtab + zz->Order.nbr_objects; for (i = 0 ; i < zz->Order.nbr_objects ; ++i) { proctab[i] = Zoltan_Get_Processor_Graph(vtxdist, zz->Num_Proc, zz->Order.rank[i] - zz->Order.start_index); sendtab[i] = zz->Order.rank[i] - zz->Order.start_index; } ierr = Zoltan_Comm_Create(&plan, zz->Order.nbr_objects, proctab, zz->Communicator, tag++, &nrecv); if (ierr != ZOLTAN_OK) { ZOLTAN_FREE(&sendgidtab); ZOLTAN_FREE(&proctab); ZOLTAN_FREE(&vtxdist); return (ierr); } ierr = Zoltan_Comm_Do(plan, tag++, (char *) sendtab, sizeof(int), (char *) recvtab); if (ierr != ZOLTAN_OK) { ZOLTAN_FREE(&sendgidtab); ZOLTAN_FREE(&proctab); ZOLTAN_FREE(&vtxdist); return (ierr); } offset = vtxdist[zz->Proc]; for (i = 0 ; i < zz->Order.nbr_objects ; ++i) { int position; position = recvtab[i]-offset; ZOLTAN_SET_GID(zz, &sendgidtab[i], &global_ids[position]); } ierr = Zoltan_Comm_Do_Reverse(plan, tag++, (char *) sendgidtab, zz->Num_GID*sizeof(int), NULL, (char *) order_ids); Zoltan_Comm_Destroy(&plan); ZOLTAN_FREE(&sendgidtab); ZOLTAN_FREE(&proctab); ZOLTAN_FREE(&vtxdist); return (ierr); }
int Zoltan_Order( ZZ *zz, /* Zoltan structure */ int num_gid_entries, /* # of entries for a global id */ int num_obj, /* Number of objects to order */ ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */ /* The application must allocate enough space */ int *rank, /* rank[i] is the rank of gids[i] */ int *iperm /* iperm[rank[i]]=i, only for sequential ordering */ ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids or enough space to store such a list * lids, a list of local ids or enough space to store such a list * Output: * num_gid_entries * num_lid_entries * gids, a list of global ids (filled in if empty on entry) * lids, a list of local ids (filled in if empty on entry) * rank, rank[i] is the global rank of gids[i] * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; int * vtxdist = NULL; ZOLTAN_ID_PTR local_gids=NULL, lids=NULL; int local_num_obj; int *local_rank = NULL, *local_iperm=NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = gcomm[0]; if (num_gid_entries != zz->Num_GID) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "num_gid_entries doesn't have the good value"); return (ZOLTAN_FATAL); } zz->Order.nbr_objects = num_obj; zz->Order.rank = rank; zz->Order.iperm = iperm; zz->Order.gids = gids; zz->Order.lids = lids; zz->Order.start = NULL; zz->Order.ancestor = NULL; zz->Order.leaves = NULL; zz->Order.nbr_leaves = 0; zz->Order.nbr_blocks = 0; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); #ifdef HAVE_MPI strncpy(opt.order_type, "DIST", MAX_PARAM_STRING_LEN); #else strncpy(opt.order_type, "SERIAL", MAX_PARAM_STRING_LEN); #endif /* HAVE_MPI */ opt.use_order_info = 0; opt.start_index = 0; opt.reorder = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type); Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index); Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); zz->Order.start_index = opt.start_index; /* * Check that the user has allocated space for the return args. */ if (!(gids && rank)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } #ifdef ZOLTAN_PARMETIS else if (!strcmp(opt.method, "NODEND")) { Order_fn = Zoltan_ParMetis_Order; } else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "GLOBAL"); } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_SCOTCH else if (!strcmp(opt.method, "SCOTCH")) { Order_fn = Zoltan_Scotch_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); /* strcpy(opt.order_type, "GLOBAL"); */ } #endif /* ZOLTAN_SCOTCH */ else { fprintf(stderr, "%s\n", opt.method); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } if (!strcmp(opt.order_type, "GLOBAL")) strcpy (opt.order_type, "DIST"); if (!strcmp(opt.order_type, "LOCAL")) strcpy (opt.order_type, "SERIAL"); strcpy(zz->Order.order_type, opt.order_type); /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /* * Call the actual ordering function. * Compute gid according to the local graph. */ if (zz->Get_Num_Obj != NULL) { local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); return (ierr); } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); return (ZOLTAN_FATAL); } local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj); local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); local_iperm = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj); ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, local_iperm, &opt); ZOLTAN_FREE(&lids); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN) { ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); Zoltan_Multifree(__FILE__, __LINE__, 3, &local_gids, &local_rank, &local_iperm); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* Compute inverse permutation if necessary */ if ((!(opt.return_args & RETURN_RANK) && (rank != NULL)) || (!(opt.return_args & RETURN_IPERM) && (iperm != NULL))) { ierr = Zoltan_Get_Distribution(zz, &vtxdist); if (ierr) { /* Error */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n"); return (ierr); } if (!(opt.return_args & RETURN_RANK) && (rank != NULL)) { /* Compute rank from iperm */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, local_iperm, local_rank, vtxdist, opt.order_type, opt.start_index); } else if (!(opt.return_args & RETURN_IPERM) && (iperm != NULL)) { /* Compute iperm from rank */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, local_rank, local_iperm, vtxdist, opt.order_type, opt.start_index); } ZOLTAN_FREE(&vtxdist); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done Invert Permutation"); /* TODO: Use directly the "graph" structure to avoid to duplicate things. */ /* I store : GNO, rank, iperm */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, (local_iperm==NULL)?0:1, local_num_obj, 0); /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj); /* Associate all the data with our xGNO */ Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, (ZOLTAN_ID_PTR) local_iperm, NULL, local_num_obj); ZOLTAN_FREE(&local_gids); ZOLTAN_FREE(&local_rank); ZOLTAN_FREE(&local_iperm); Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)rank, (ZOLTAN_ID_PTR)iperm, NULL, num_obj, NULL); Zoltan_DD_Destroy(&dd); ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < num_obj; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)])); printf(", rank = %3d\n", rank[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } ZOLTAN_TRACE_EXIT(zz, yo); if (ierr) return (ierr); else return (ZOLTAN_OK); }
static int Zoltan_Oct_migreg_migrate_regions(ZZ *zz, Region *regions, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids, int *npids, int nregions, int *c2) { char *yo = "Zoltan_Oct_migreg_migrate_regions"; int i; /* index counter */ int ierr = ZOLTAN_OK; int n_import; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */ Region *import_objs = NULL; /* Array of import objects used to request the objs from other processors. */ ZOLTAN_ID_PTR import_gids = NULL; /* Array of global IDs of import_objs. */ ZOLTAN_ID_PTR import_lids = NULL; /* Array of local IDs of import_objs. */ int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; ierr = Zoltan_Comm_Create(&comm_plan, nregions, npids, zz->Communicator, MIGMIGREGCommCreate, &n_import); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Create"); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } *c2 = n_import; if (n_import > 0) { import_objs = (Region *) ZOLTAN_MALLOC(n_import * sizeof(Region)); import_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, n_import); import_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, n_import); if (!import_objs || !import_gids || (num_lid_entries && !import_lids)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } } ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo, (char *) regions, sizeof(Region), (char *) import_objs); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); return (ierr); } ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-1, (char *) gids, sizeof(ZOLTAN_ID_TYPE)*num_gid_entries, (char *) import_gids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); return (ierr); } if (num_lid_entries > 0) { ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-2, (char *) lids, sizeof(ZOLTAN_ID_TYPE)*num_lid_entries, (char *) import_lids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); return (ierr); } } for (i=0; i<n_import; i++) { import_objs[i].Global_ID = &(import_gids[i*num_gid_entries]); import_objs[i].Local_ID = (num_lid_entries ? &(import_lids[i*num_lid_entries]) : NULL); Zoltan_Oct_insert_orphan(zz, import_objs[i]); } ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); ierr = Zoltan_Comm_Destroy(&comm_plan); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } return ierr; }
int Zoltan_Oct_migreg_migrate_orphans(ZZ *zz, pRegion RegionList, int nregions, int level, Map *array, int *c1, int *c2) { int i, j, k; /* index counters */ pRegion ptr; /* region in the mesh */ COORD origin; /* centroid coordinate information */ pRegion *regions = NULL; /* an array of regions */ int *npids = NULL; Region *regions2 = NULL; /* an array of regions */ int *npids2 = NULL; int nreg; /* number of regions */ COORD min, /* minimum bounds of an octant */ max; /* maximum bounds of an octant */ COORD cmin, /* minimum bounds of a child octant */ cmax; /* maximum bounds of a child octant */ COORD rmin, /* minimum bounds of a remote octant */ rmax; /* maximum bounds of a remote octant */ int new_num; int n; int dir = 0; pRList RootList; pOctant RootOct; OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure); char *yo = "Zoltan_Oct_migreg_migrate_orphans_static"; int ierr = ZOLTAN_OK; ZOLTAN_ID_PTR gids2, lids2; int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; if(nregions > 0) { /* create the array of messages to be sent to other processors */ /* Array = (Message *) ZOLTAN_MALLOC(nregions * sizeof(Message)); */ if((regions = (pRegion *) ZOLTAN_MALLOC(nregions * sizeof(pRegion))) == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if((npids = (int *) ZOLTAN_MALLOC(nregions * sizeof(int))) == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(®ions); return ZOLTAN_MEMERR; } } ptr = RegionList; n = nreg = 0; while((ptr != NULL) && (nregions > 0)) { if(ptr->attached == 1) { /* if region already attached to an octant, then skip to next region */ ptr = ptr->next; continue; } /* region not attached, have to find which processor to send to */ j=0; dir = 0; vector_set(min, OCT_info->OCT_gmin); vector_set(max, OCT_info->OCT_gmax); /* * for each level of refinement, find which child region belongs to. * translate which child to which entry in map array. */ for(i=0; i<level; i++) { Zoltan_Oct_bounds_to_origin(min, max, origin); if(OCT_info->OCT_dimension == 2) j = j * 4; else j = j * 8; k = Zoltan_Oct_child_which(OCT_info,origin, ptr->Coord); new_num = Zoltan_Oct_convert_idx_from_map(OCT_info, dir, k); dir = Zoltan_Oct_get_child_dir(OCT_info, dir, new_num); j += new_num; Zoltan_Oct_child_bounds(min, max, origin, k, cmin, cmax); vector_set(min, cmin); vector_set(max, cmax); } /* inform message which processor to send to */ npids[n] = array[j].npid; RootList = array[j].list; while((RootOct = RL_nextRootOctant(&RootList))) { Zoltan_Oct_bounds(RootOct,rmin,rmax); if (Zoltan_Oct_in_box_closure(OCT_info, ptr->Coord ,rmin, rmax)) { npids[n] = RootOct->npid; break; } } if((npids[n] != -1) && (npids[n] != zz->Proc)) { Zoltan_Oct_copy_info(zz, ptr, &(regions[n++])); } else { Zoltan_Oct_insert_orphan(zz, *ptr); } nreg++; /* increment region counter */ ptr = ptr->next; /* look at next region */ } /* * if regions looked at != number of regions in region list, * then there is an error */ if (nreg!=nregions) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "regions found != to expected number of regions"); return ZOLTAN_FATAL; } regions2 = (Region *) ZOLTAN_MALLOC(n * sizeof(Region)); gids2 = ZOLTAN_MALLOC_GID_ARRAY(zz, n); lids2 = ZOLTAN_MALLOC_LID_ARRAY(zz, n); npids2 = (int *) ZOLTAN_MALLOC(n * sizeof(int)); for(i=0; i<n; i++) { npids2[i] = npids[i]; vector_set(regions2[i].Coord, regions[i]->Coord); regions2[i].Weight = regions[i]->Weight; regions2[i].Global_ID = &(gids2[i*num_gid_entries]); regions2[i].Local_ID = (num_lid_entries ? &(lids2[i*num_lid_entries]) : NULL); ZOLTAN_SET_GID(zz, &(gids2[i*num_gid_entries]), regions[i]->Global_ID); ZOLTAN_SET_LID(zz, &(lids2[i*num_lid_entries]), regions[i]->Local_ID); regions2[i].Proc = regions[i]->Proc; regions2[i].attached = 0; } *c1 = n; /* migrate the orphan regions according to the message array */ Zoltan_Oct_migreg_migrate_regions(zz, regions2, gids2, lids2, npids2, n, c2); for (i=0; i < n; i++) { ZOLTAN_FREE(&(regions[i]->Global_ID)); ZOLTAN_FREE(&(regions[i]->Local_ID)); ZOLTAN_FREE(&(regions[i])); } ZOLTAN_FREE(®ions); ZOLTAN_FREE(&npids); ZOLTAN_FREE(®ions2); ZOLTAN_FREE(&gids2); ZOLTAN_FREE(&lids2); ZOLTAN_FREE(&npids2); return ierr; }
int Zoltan_Order ( struct Zoltan_Struct *zz, int num_gid_entries, int num_obj, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR permuted_global_ids ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids. * num_gid_entries * Output: * permuted_global_ids * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; ZOLTAN_ID_PTR local_gids=NULL, lids=NULL; int local_num_obj; int *local_rank = NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = gcomm[0]; if (num_gid_entries != zz->Num_GID) { char msg[253]; sprintf(msg, "num_gid_entries=%d is not equal to parameter setting " "NUM_GID_ENTRIES=%d\n", num_gid_entries, zz->Num_GID); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); return (ZOLTAN_FATAL); } zz->Order.nbr_objects = num_obj; zz->Order.start = NULL; zz->Order.ancestor = NULL; zz->Order.leaves = NULL; zz->Order.nbr_leaves = 0; zz->Order.nbr_blocks = 0; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ #ifdef HAVE_MPI strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); strcpy(zz->Order.order_type, "GLOBAL"); #else strncpy(opt.method, "METIS", MAX_PARAM_STRING_LEN); strcpy(zz->Order.order_type, "LOCAL"); #endif /* HAVE_MPI */ opt.use_order_info = 0; opt.start_index = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); /* * Check that the user has allocated space for the return args. */ if (num_obj && !(gids && permuted_global_ids)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } else if (!strcmp(opt.method, "LOCAL_HSFC")) { Order_fn = Zoltan_LocalHSFC_Order; strcpy(zz->Order.order_type, "LOCAL"); /*MMW, not sure about this*/ } #ifdef ZOLTAN_PARMETIS else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; strcpy(zz->Order.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; strcpy(zz->Order.order_type, "GLOBAL"); } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_SCOTCH else if (!strcmp(opt.method, "SCOTCH")) { Order_fn = Zoltan_Scotch_Order; strcpy(zz->Order.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PTSCOTCH")) { Order_fn = Zoltan_Scotch_Order; strcpy(zz->Order.order_type, "GLOBAL"); } #endif /* ZOLTAN_SCOTCH */ #ifdef ZOLTAN_HUND else if (!strcasecmp(opt.method, "HUND")) { ierr = Zoltan_HUND(zz, num_gid_entries, num_obj, gids, permuted_global_ids, NULL); goto End; } #endif /* ZOLTAN_HUND */ else { fprintf(stderr, "%s\n", opt.method); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* TODO : Ask why useful ! */ /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /************************************ * Check for required query function ************************************/ if (zz->Get_Num_Obj != NULL) { local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); return (ierr); } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); return (ZOLTAN_FATAL); } /* TODO allocate all this stuff with the graph */ local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj); local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj); /* * Call the actual ordering function. * Compute gid according to the local graph. */ ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, NULL, &opt); ZOLTAN_FREE(&lids); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN) { ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); Zoltan_Multifree(__FILE__, __LINE__, 2, &local_gids, &local_rank); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* TODO: Use directly the "graph" structure to avoid to duplicate things. */ /* TODO: At this time, I consider rank == permuted_global_ids */ /* I store : GNO, rank, permuted GID */ /* MMW: perhaps don't ever use graph here since we need to support geometric orderings, otherwise need if/else */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, 0, local_num_obj, 0); /* Hope a linear assignment will help a little */ if (local_num_obj) Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj); /* Associate all the data with our xGNO */ Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, NULL, NULL, local_num_obj); ZOLTAN_FREE(&local_gids); ZOLTAN_FREE(&local_rank); Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)permuted_global_ids, NULL, NULL, num_obj, NULL); Zoltan_DD_Destroy(&dd); ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < num_obj; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)])); printf(", rank = %3d\n", permuted_global_ids[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } #ifdef ZOLTAN_HUND End: #endif /*ZOLTAN_HUND*/ ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_PHG_2ways_hyperedge_partition ( ZZ *zz, /* Input : Zoltan data structure */ HGraph *hg, Partition parts, Zoltan_PHG_Tree *tree, struct Zoltan_DD_Struct * gnoToGID, struct Zoltan_DD_Struct **dd, int *numParts, int **sizeParts ) { int ierr = ZOLTAN_OK; char *yo = "Zoltan_PHG_2ways_hyperedge_partition"; int nEdge, hEdge; int *interval; int *partnumber = NULL; int tree_size; int *rowpart =NULL; int *rowGNO = NULL; ZOLTAN_ID_PTR rowGID=NULL; int index; int offset; ZOLTAN_TRACE_ENTER(zz, yo); nEdge = hg->nEdge; fprintf (stderr, "HG (%d %d x %d) : %d %d\n", hg->comm->myProc, hg->comm->myProc_x, hg->comm->myProc_y, hg->nVtx, nEdge); interval = (int*)ZOLTAN_MALLOC(nEdge*2*sizeof(int)); if ((nEdge > 0 ) && (interval == NULL)) MEMORY_ERROR; tree_size = get_tree_size(tree) + 1; for (index = 0 ; index < nEdge ; ++index){ SET_MIN_NODE(interval, index, tree_size); SET_MAX_NODE(interval, index, -1); } /* Update interval with the local knowledge */ /* XXX: I loop on the hyperedges, as I think it's more cache friendly * and it allows me to discoupled the computation if a non blocking MPI_Reduce is * available */ for (hEdge = 0 ; hEdge < nEdge ; ++hEdge){ int part; int max = -1; /* Trick : we use the initialized values */ int min = tree_size + 1; for (index = hg->hindex[hEdge] ; index < hg->hindex[hEdge+1] ; ++ index) { part = parts[hg->hvertex[index]]; max = MAX(max, part); min = MIN(min, part); } SET_MIN_NODE(interval, hEdge, min); SET_MAX_NODE(interval, hEdge, max); } /* Update results to view the complete hyperedges */ Zoltan_AllReduceInPlace (interval, 2*nEdge, MPI_INT, MPI_MAX, hg->comm->row_comm); /* Now I have to compute the partition of hyperedges according to the "interval" * and the tree */ /* First, compute the partition number corresponding to the nodes in the tree */ partnumber = compute_part_number(tree); if (partnumber == NULL) { ierr = ZOLTAN_FATAL; goto End; } (*numParts) = get_tree_size(tree); rowpart = (int*) ZOLTAN_MALLOC(nEdge*sizeof(int)); if ((nEdge > 0) && (rowpart == NULL)) MEMORY_ERROR; rowGNO = (int*) ZOLTAN_MALLOC(nEdge*sizeof(int)); if ((nEdge > 0) && (rowGNO == NULL)) MEMORY_ERROR; (*sizeParts) = (int*)ZOLTAN_CALLOC((*numParts), sizeof(int)); if (*numParts && (*sizeParts) == NULL) MEMORY_ERROR; offset = hg->dist_y[hg->comm->myProc_y]; /* Then we search we is the hyperedge in the tree */ for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) { int node; node = find_interval_in_tree(tree, interval+2*hEdge); rowpart[hEdge] = partnumber[node]; (*sizeParts)[rowpart[hEdge]] ++; rowGNO[hEdge] = EDGE_LNO_TO_GNO(hg, hEdge); #if 0 fprintf (stderr, "%d : %d (%d : %d - %d)\n", rowGNO[hEdge], rowpart[hEdge], node, -interval[2*hEdge], interval[2*hEdge+1]); #endif } partnumber += 1; ZOLTAN_FREE(&partnumber); ZOLTAN_FREE(&interval); /* Compute number of elements per parts */ /* TODO: support processor which are not part of the distribution */ /* Update results to view the complete hyperedges */ Zoltan_AllReduceInPlace ((*sizeParts), (*numParts), MPI_INT, MPI_SUM, hg->comm->col_comm); /* Export results to data directory */ /* First, get the GIDs of our edges */ rowGID = ZOLTAN_MALLOC_GID_ARRAY(zz, nEdge); if (nEdge && rowGID == NULL) MEMORY_ERROR; ierr = Zoltan_DD_Find (gnoToGID , (ZOLTAN_ID_PTR)rowGNO, rowGID, NULL, NULL, nEdge, NULL); ZOLTAN_FREE(&rowGNO); ierr = Zoltan_DD_Create (dd, zz->Communicator, zz->Num_GID, 1, 0, nEdge, 0); CHECK_IERR; /* Make our new numbering public */ Zoltan_DD_Update (*dd, (ZOLTAN_ID_PTR)rowGID, (ZOLTAN_ID_PTR) rowpart, NULL, NULL, nEdge); #ifdef CEDRIC_PRINT for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) { fprintf (stderr, "%d : %d\n", rowGID[hEdge], rowpart[hEdge]); } #endif End: ZOLTAN_FREE(&rowGID); ZOLTAN_FREE(&rowGNO); ZOLTAN_FREE(&rowpart); if (partnumber != NULL) partnumber += 1; ZOLTAN_FREE(&partnumber); ZOLTAN_FREE(&interval); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
/* * Each processor: * owns a set of pins (nonzeros) * may provide some edge weights * * We assume that no two processes will supply the same pin. * But more than one process may supply pins for the same edge. */ static int matrix_get_edges(ZZ *zz, Zoltan_matrix *matrix, ZOLTAN_ID_PTR *yGID, ZOLTAN_ID_PTR *pinID, int nX, ZOLTAN_ID_PTR *xGID, ZOLTAN_ID_PTR *xLID, int **xGNO, float **xwgt) { static char *yo = "Zoltan_Matrix_Build"; int ierr = ZOLTAN_OK; int hypergraph_callbacks = 0, graph_callbacks = 0; int *nbors_proc = NULL; /* Pointers are global for the function to ensure proper free */ int *edgeSize = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Get_HG_Size_CS && zz->Get_HG_CS) { hypergraph_callbacks = 1; } if ((zz->Get_Num_Edges != NULL || zz->Get_Num_Edges_Multi != NULL) && (zz->Get_Edge_List != NULL || zz->Get_Edge_List_Multi != NULL)) { graph_callbacks = 1; } if (graph_callbacks && hypergraph_callbacks){ /* if (hgraph_model == GRAPH) */ /* hypergraph_callbacks = 0; */ graph_callbacks = 1; /* I prefer graph (allow to do "inplace") ! */ } if (hypergraph_callbacks) { matrix->redist = 1; if (matrix->opts.speed == MATRIX_FULL_DD) ZOLTAN_FREE(xGID); else *xGID = NULL; ZOLTAN_FREE(xLID); ZOLTAN_FREE(xGNO); ierr = Zoltan_Hypergraph_Queries(zz, &matrix->nY, &matrix->nPins, yGID, &matrix->ystart, pinID); CHECK_IERR; matrix->yend = matrix->ystart + 1; } else if (graph_callbacks) { int max_edges = 0; int vertex; int numGID, numLID; matrix->opts.enforceSquare = 1; matrix->nY = nX; /* It is square ! */ matrix->yGNO = *xGNO; *xGNO = NULL; *yGID = NULL; matrix->ywgtdim = zz->Obj_Weight_Dim; *xwgt = NULL; numGID = zz->Num_GID; numLID = zz->Num_LID; /* TODO : support local graphs */ /* TODO : support weights ! */ /* Get edge data */ Zoltan_Get_Num_Edges_Per_Obj(zz, matrix->nY, *xGID, *xLID, &edgeSize, &max_edges, &matrix->nPins); (*pinID) = ZOLTAN_MALLOC_GID_ARRAY(zz, matrix->nPins); nbors_proc = (int *)ZOLTAN_MALLOC(matrix->nPins * sizeof(int)); if (matrix->nPins && ((*pinID) == NULL || nbors_proc == NULL)) MEMORY_ERROR; matrix->pinwgt = (float*)ZOLTAN_MALLOC(matrix->nPins*matrix->pinwgtdim*sizeof(float)); if (matrix->nPins && matrix->pinwgtdim && matrix->pinwgt == NULL) MEMORY_ERROR; if (zz->Get_Edge_List_Multi) { zz->Get_Edge_List_Multi(zz->Get_Edge_List_Multi_Data, numGID, numLID, matrix->nY, *xGID, *xLID, edgeSize, (*pinID), nbors_proc, matrix->pinwgtdim, matrix->pinwgt, &ierr); } else { int edge; for (vertex = 0, edge = 0 ; vertex < matrix->nY ; ++vertex) { zz->Get_Edge_List(zz->Get_Edge_List_Data, numGID, numLID, (*xGID)+vertex*numGID, (*xLID)+vertex*numLID, (*pinID)+edge*numGID, nbors_proc+edge, matrix->pinwgtdim, matrix->pinwgt+edge*matrix->pinwgtdim, &ierr); edge += edgeSize[vertex]; } } CHECK_IERR; /* Not Useful anymore */ ZOLTAN_FREE(xLID); if (matrix->opts.speed == MATRIX_FULL_DD) ZOLTAN_FREE(xGID); else *xGID = NULL; ZOLTAN_FREE(&nbors_proc); /* Now construct CSR indexing */ matrix->ystart = (int*) ZOLTAN_MALLOC((matrix->nY+1)*sizeof(int)); if (matrix->ystart == NULL) MEMORY_ERROR; matrix->ystart[0] = 0; matrix->yend = matrix->ystart + 1; for (vertex = 0 ; vertex < matrix->nY ; ++vertex) matrix->ystart[vertex+1] = matrix->ystart[vertex] + edgeSize[vertex]; } else { FATAL_ERROR ("You have to define Hypergraph or Graph queries"); } if (matrix->opts.enforceSquare) { matrix->globalY = matrix->globalX; matrix->ddY = matrix->ddX; matrix->ywgtdim = zz->Obj_Weight_Dim; } End: ZOLTAN_FREE(&edgeSize); ZOLTAN_FREE(&nbors_proc); ZOLTAN_FREE(xLID); ZOLTAN_FREE(xGID); ZOLTAN_FREE(xGNO); ZOLTAN_FREE(xwgt); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
static int Zoltan_Reftree_Sum_Weights(ZZ *zz) { /* * Function to sum the weights in the refinement tree. On input the * refinement tree should be valid and have weight set. On output the * values in summed_weight at each node is the sum of the weights in the * subtree with that node as root. * This function also sets assigned_to_me for interior nodes to be * 1 if the entire subtree is assigned to this processor * 0 if none of the subtree is assigned to this processor * -1 if some of the subtree is assigned to this processor */ char *yo = "Zoltan_Reftree_Sum_Weights"; ZOLTAN_REFTREE *root; /* Root of the refinement tree */ int wdim; /* Dimension of the weight array */ int i,j; /* loop counters */ int count; /* counter */ ZOLTAN_ID_PTR leaf_list = NULL; /* leaves for which some proc requests weight */ ZOLTAN_ID_PTR all_leaflist = NULL; /* leaf_list from all processors */ int reqsize; /* length of leaf_list */ int *reqsize_all; /* reqsize from all processors */ int sum_reqsize; /* sum of all reqsize */ int *displs; /* running sum of all reqsize */ int my_start; /* position in leaf_list of this proc's list */ int nproc; /* number of processors */ ZOLTAN_REFTREE *node; /* a node in the refinement tree */ struct Zoltan_Reftree_hash_node **hashtab; /* hash table */ int hashsize; /* dimension of hash table */ float *send_float; /* sending message of floats */ float *req_weights; /* the requested weights */ int num_gid_entries = zz->Num_GID; /* Number of array entries in a global ID */ ZOLTAN_TRACE_ENTER(zz, yo); /* * set the root and hash table */ root = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->reftree_root; if (root == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Refinement tree not defined."); ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_FATAL); } hashtab = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->hash_table; hashsize = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->hash_table_size; /* * Determine the dimension of the weight array */ if (zz->Obj_Weight_Dim == 0) { wdim = 1; } else { wdim = zz->Obj_Weight_Dim; } /* * In the first pass, sum the weights of the nodes that are assigned to * this processor, and count the leaves that are not. */ count = 0; for (i=0; i<root->num_child; i++) { Zoltan_Reftree_Sum_My_Weights(zz,&(root->children[i]),&count,wdim); } root->assigned_to_me = -1; /* * Make a list of the leaves that are not assigned to this processor */ if (count == 0) leaf_list = ZOLTAN_MALLOC_GID(zz); else leaf_list = ZOLTAN_MALLOC_GID_ARRAY(zz, count); if (leaf_list == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_MEMERR); } count = 0; Zoltan_Reftree_List_Other_Leaves(zz, root,leaf_list,&count); /* * Get the unknown leaf weights from other processors. */ nproc = zz->Num_Proc; reqsize = count; /* * Build a list of all processor's request list by concatinating them in * the order of the processor ranks */ /* * Determine the request size of all processors */ reqsize_all = (int *)ZOLTAN_MALLOC(nproc*sizeof(int)); displs = (int *)ZOLTAN_MALLOC(nproc*sizeof(int)); if (reqsize_all == NULL || displs == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); Zoltan_Multifree(__FILE__, __LINE__, 3, &displs, &reqsize_all, &leaf_list); ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_MEMERR); } MPI_Allgather((void *)&reqsize,1,MPI_INT,(void *)reqsize_all,1,MPI_INT, zz->Communicator); displs[0] = 0; for (i=1; i<nproc; i++) displs[i] = displs[i-1]+reqsize_all[i-1]; sum_reqsize = displs[nproc-1] + reqsize_all[nproc-1]; my_start = displs[zz->Proc]; /* * If sum_reqsize is 0, nothing needs to be communciated */ if (sum_reqsize == 0) { Zoltan_Multifree(__FILE__, __LINE__, 3, &displs, &reqsize_all, &leaf_list); } else { /* * Gather the request list from all processors */ all_leaflist = ZOLTAN_MALLOC_GID_ARRAY(zz, sum_reqsize); if (all_leaflist == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); Zoltan_Multifree(__FILE__, __LINE__, 4, &all_leaflist, &displs, &reqsize_all, &leaf_list); ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_MEMERR); } /* KDDKDD Changed MPI_BYTE to ZOLTAN_ID_MPI_TYPE */ /* Account for number of array entries in an ID. */ for (i=0; i<nproc; i++) { reqsize_all[i] = reqsize_all[i]*num_gid_entries; displs[i] = displs[i]*num_gid_entries; } MPI_Allgatherv((void *)leaf_list,reqsize*num_gid_entries,ZOLTAN_ID_MPI_TYPE, (void *)all_leaflist,reqsize_all,displs,ZOLTAN_ID_MPI_TYPE, zz->Communicator); ZOLTAN_FREE(&displs); ZOLTAN_FREE(&leaf_list); for (i=0; i<nproc; i++) reqsize_all[i] = reqsize_all[i]/num_gid_entries; /* * Create a list with the partial sums this processor has */ send_float = (float *) ZOLTAN_MALLOC(sizeof(float)*wdim*sum_reqsize); if (send_float == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); Zoltan_Multifree(__FILE__, __LINE__, 3, &send_float, &all_leaflist, &reqsize_all); ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_MEMERR); } for (i=0; i<sum_reqsize; i++) { node = Zoltan_Reftree_hash_lookup(zz, hashtab, &(all_leaflist[i*num_gid_entries]), hashsize); if (node == NULL) for (j=0; j<wdim; j++) send_float[i*wdim+j] = 0.0; else for (j=0; j<wdim; j++) send_float[i*wdim+j] = node->my_sum_weight[j]; } /* * Sum the weights over all the processors */ if (reqsize == 0) req_weights = (float *) ZOLTAN_MALLOC(sizeof(float)*wdim); else req_weights = (float *) ZOLTAN_MALLOC(sizeof(float)*wdim*reqsize); if (req_weights == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); Zoltan_Multifree(__FILE__, __LINE__, 4, &req_weights, &send_float, &all_leaflist, &reqsize_all); ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_MEMERR); } MPI_Reduce_scatter((void *)send_float, (void *)req_weights, reqsize_all, MPI_FLOAT, MPI_SUM, zz->Communicator); ZOLTAN_FREE(&send_float); ZOLTAN_FREE(&reqsize_all); /* * Set the weights this processor requested */ for (i=0; i<count; i++) { node = Zoltan_Reftree_hash_lookup(zz, hashtab, &(all_leaflist[(i+my_start)*num_gid_entries]), hashsize); for (j=0; j<wdim; j++) node->summed_weight[j] = req_weights[i*wdim+j]; } ZOLTAN_FREE(&req_weights); ZOLTAN_FREE(&all_leaflist); } /* * All the leaves now have summed_weight set. * Sum the weights throughout the tree. */ Zoltan_Reftree_Sum_All_Weights(zz,root,wdim); ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_OK); }
int Zoltan_Matrix_Sym(ZZ* zz, Zoltan_matrix *matrix, int bipartite) { static char *yo = "Zoltan_Matrix_Sym"; int ierr = ZOLTAN_OK; Zoltan_Arc *tr_tab = NULL; int i, j, cnt; ZOLTAN_ID_PTR yGID = NULL; int *ypid=NULL; float *pinwgt=NULL; int * ybipart = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (bipartite || !matrix->opts.enforceSquare) { matrix->bipartite = 1; matrix->redist = 1; } if (matrix->ywgtdim != zz->Obj_Weight_Dim) FATAL_ERROR("Cannot form bipartite graph: vertex and edge weights are not consistant"); matrix->opts.symmetrize = 1; /* Update the data directories */ tr_tab = (Zoltan_Arc*) ZOLTAN_MALLOC(sizeof(Zoltan_Arc)*(matrix->nPins*2+matrix->nY)); if (matrix->nPins && tr_tab == NULL) MEMORY_ERROR; pinwgt = (float*)ZOLTAN_MALLOC((matrix->nPins*2+matrix->nY)*matrix->pinwgtdim*sizeof(float)); for (i = 0 ; i < 2 ; ++i) /* Copy pin weights */ memcpy(pinwgt + i*matrix->nPins*matrix->pinwgtdim*sizeof(float), matrix->pinwgt, matrix->nPins*matrix->pinwgtdim*sizeof(float)); for (i=0, cnt = 0 ; i < matrix->nY ; ++i) { for (j = matrix->ystart[i] ; j < matrix->yend[i] ; ++j) { tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */ tr_tab[cnt].GNO[1] = matrix->pinGNO[j]; memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim, matrix->pinwgtdim*sizeof(float)); cnt ++; tr_tab[cnt].GNO[0] = matrix->pinGNO[j]; /* Symmetric arc */ tr_tab[cnt].GNO[1] = matrix->yGNO[i] + bipartite*matrix->globalX; /* new ordering */ memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim, matrix->pinwgtdim*sizeof(float)); cnt ++; } if (matrix->ystart[i] == matrix->yend[i]) { /* Singleton */ tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */ tr_tab[cnt].GNO[1] = -1; cnt ++; } } ZOLTAN_FREE(&matrix->pinwgt); Zoltan_Matrix_Remove_DupArcs(zz, cnt, tr_tab, pinwgt, matrix); ZOLTAN_FREE(&tr_tab); ZOLTAN_FREE(&pinwgt); if (bipartite) { int endX; int * yGNO = NULL; /* Update data directories */ yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, matrix->nY); ypid = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int)); ybipart = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int)); for (endX = 0 ; endX < matrix->nY ; ++endX) { if (matrix->yGNO[endX] >= matrix->globalX) break; ybipart[endX] = 0; } /* Get Informations about X */ Zoltan_DD_Find (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, NULL, endX, NULL); yGNO = (int*)ZOLTAN_MALLOC(endX*sizeof(int)); for (i = endX ; i < matrix->nY ; ++i) { yGNO[i-endX] = matrix->yGNO[i] - matrix->globalX; /* TODO: add a something to have the correct ypid */ ybipart[endX] = 1; } /* Get Informations about Y */ Zoltan_DD_Find (matrix->ddY, (ZOLTAN_ID_PTR)yGNO, yGID + endX*zz->Num_GID, NULL, NULL, matrix->nY-endX, NULL); if (matrix->ddY != matrix->ddX) Zoltan_DD_Destroy (&matrix->ddY); Zoltan_DD_Destroy (&matrix->ddX); matrix->globalX += matrix->globalY; matrix->globalY = matrix->globalX; /* I store : xGNO, xGID, xpid, bipart */ ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, 1, zz->Num_GID, 1, matrix->globalX/zz->Num_Proc, 0); matrix->ddY = matrix->ddX; /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc); /* Associate all the data with our xyGNO */ Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, ybipart, matrix->nY); } End: ZOLTAN_FREE(&ybipart); ZOLTAN_FREE(&ypid); ZOLTAN_FREE(&pinwgt); ZOLTAN_FREE(&yGID); ZOLTAN_FREE(&tr_tab); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }