/* * void Zoltan_Oct_insert_orphan(pRegion region) * * Insert orphan regions migrated from off processors, or to insert * regions that lie on the boundary. */ static int Zoltan_Oct_insert_orphan(ZZ *zz, Region reg) { pRList RootList; /* list of all local roots */ pOctant RootOct; int rflag; /* flag to indicate region fits in octant */ int i, j; /* index counters */ double upper, /* upper bounds of the octant */ lower; /* lower bounds of the octant */ OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure); char *yo = "Zoltan_Oct_insert_orphan"; int ierr = ZOLTAN_OK; if (OCT_info->OCT_dimension == 2) i = 2; /* ignore z coordinates */ else i = 3; rflag = 0; RootList = Zoltan_Oct_POct_localroots(OCT_info); /* get a list all local roots */ while((RootOct = RL_nextRootOctant(&RootList))) { rflag = 1; for (j=0; j<i; j++) { lower = RootOct->min[j]; upper = RootOct->max[j]; if (reg.Coord[j]<lower || reg.Coord[j]>upper) { /* if region coord lie outside bounds, then cannot fit */ rflag = 0; break; } } if(rflag == 1) { /* region fits inside octant */ /* found a place to insert region */ Zoltan_Oct_subtree_insert(zz, RootOct, ®); return ierr; } } ierr = ZOLTAN_WARN; ZOLTAN_TRACE_DETAIL(zz, yo, "could not insert region"); fprintf(stderr,"%s failed to insert %f %f %f on proc %d\n",yo, reg.Coord[0], reg.Coord[1], reg.Coord[2], zz->Proc); RootList = Zoltan_Oct_POct_localroots(OCT_info); RL_printRootOctants(RootList); return ierr; }
int Zoltan_Migrate( ZZ *zz, /* Zoltan structure. */ int num_import, /* Number of non-local objects assigned to the processor in the new decomposition. */ ZOLTAN_ID_PTR import_global_ids, /* Array of global IDs for non-local objects assigned to this processor in the new decomposition; this field can be NULL if the application doesn't provide import IDs.*/ ZOLTAN_ID_PTR import_local_ids, /* Array of local IDs for non-local objects assigned to the processor in the new decomposition; this field can be NULL if the application does not provide import IDs. */ int *import_procs, /* Array of processor IDs of processors owning the non-local objects that are assigned to this processor in the new decomposition; this field can be NULL if the application does not provide import IDs. */ int *import_to_part, /* Array of partition numbers to which imported objects should be assigned. */ int num_export, /* Number of objs to be exported to other processors to establish the new decomposition. */ ZOLTAN_ID_PTR export_global_ids, /* Array of global IDs of objects to be exported to other processors to establish the new decomposition. */ ZOLTAN_ID_PTR export_local_ids, /* Array of local IDs of objects to be exported to other processors to establish the new decomposition. */ int *export_procs, /* Array of processor IDs to which objects will be exported to establish the new decomposition. */ int *export_to_part /* Array of partition numbers to which exported objects should be assigned. */ ) { /* * Routine to help perform migration. If migration pre-processing routine * (ZOLTAN_PRE_MIGRATE_FN) is specified, this routine first calls that fn. * It then calls a function to obtain the size of the migrating objects * (ZOLTAN_OBJ_SIZE_FN). The routine next calls an application-specified * object packing routine (ZOLTAN_PACK_OBJ_FN) for each object * to be exported. It develops the needed communication map to move the * objects to other processors. It performs the communication according * to the map, and then calls an application-specified object unpacking * routine (ZOLTAN_UNPACK_OBJ_FN) for each object imported. */ char *yo = "Zoltan_Migrate"; int num_gid_entries, num_lid_entries; /* lengths of global & local ids */ int *sizes = NULL; /* sizes (in bytes) of the object data for export. */ int id_size; /* size (in bytes) of ZOLTAN_GID + padding for alignment */ int tag_size; /* size (in bytes) of ZOLTAN_GID + one int (for message size) */ char *export_buf = NULL; /* buffer for packing export data. */ char *import_buf = NULL; /* buffer for receiving imported data. */ char *tmp; /* temporary pointer into buffers. */ int i; /* loop counter. */ int tmp_size; /* size of a single object's data. */ int *idx = NULL; /* index used for multi-fn packs and unpacks. */ int idx_cnt = 0; /* index counter for idx array. */ ZOLTAN_ID_PTR tmp_id = NULL; /* pointer to storage for a global ID in comm buf */ ZOLTAN_ID_PTR lid; /* temporary pointer to a local ID; used to pass NULL to query functions when NUM_LID_ENTRIES=0. */ ZOLTAN_COMM_OBJ *imp_plan = NULL; /* Comm obj built from import lists. */ ZOLTAN_COMM_OBJ *exp_plan = NULL; /* Comm obj built from export lists. */ int msgtag, msgtag2; /* Tags for communication routines */ int total_send_size; /* Total size of outcoming message (in #items) */ int total_recv_size; /* Total size of incoming message (in #items) */ int aligned_int; /* size of an int padded for alignment */ int dest; /* temporary destination partition. */ int include_parts = 0; /* flag indicating whether partition info is provided */ int ierr = ZOLTAN_OK; int actual_num_exp = 0; int actual_exp_allocated = 0; ZOLTAN_ID_PTR actual_exp_gids = NULL; /* Arrays containing only objs to */ ZOLTAN_ID_PTR actual_exp_lids = NULL; /* actually be packed. Objs that */ int *actual_exp_procs = NULL; /* are changing partition but not */ int *actual_exp_to_part = NULL; /* processor may not be included. */ int actual_num_imp = 0; int actual_imp_allocated = 0; ZOLTAN_ID_PTR actual_imp_gids = NULL; /* Arrays containing only objs to */ ZOLTAN_ID_PTR actual_imp_lids = NULL; /* actually be imported. Objs that */ int *actual_imp_procs = NULL; /* are changing partition but not */ int *actual_imp_to_part = NULL; /* processor may not be included. */ ZOLTAN_TRACE_ENTER(zz, yo); /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { goto End; } /* * Check that all procs use the same id types. */ ierr = check_input(zz, ((num_export >= 0 && export_to_part) || (num_import >= 0 && import_to_part)), &include_parts); if (ierr != ZOLTAN_OK) goto End; num_gid_entries = zz->Num_GID; num_lid_entries = zz->Num_LID; /* * Check that all necessary query functions are available. */ if (zz->Get_Obj_Size == NULL && zz->Get_Obj_Size_Multi == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a " "ZOLTAN_OBJ_SIZE_FN or ZOLTAN_OBJ_SIZE_MULTI_FN function " "to use the migration-help tools."); ierr = ZOLTAN_FATAL; goto End; } if (zz->Pack_Obj == NULL && zz->Pack_Obj_Multi == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a " "ZOLTAN_PACK_OBJ_FN or ZOLTAN_PACK_OBJ_MULTI_FN function " "to use the migration-help tools."); ierr = ZOLTAN_FATAL; goto End; } if (zz->Unpack_Obj == NULL && zz->Unpack_Obj_Multi == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a " "ZOLTAN_UNPACK_OBJ_FN or ZOLTAN_UNPACK_OBJ_MULTI_FN function " "to use the migration-help tools."); ierr = ZOLTAN_FATAL; goto End; } if (num_export >= 0) { /* Build the actual export arrays */ ierr = actual_arrays(zz, num_gid_entries, num_lid_entries, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &actual_num_exp, &actual_exp_gids, &actual_exp_lids, &actual_exp_procs, &actual_exp_to_part, &actual_exp_allocated); if (ierr < 0) goto End; /* Compute communication map based on actual exports. */ msgtag = 32767; ierr = Zoltan_Comm_Create(&exp_plan, actual_num_exp, actual_exp_procs, zz->Communicator, msgtag, &actual_num_imp); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create."); goto End; } } else if (num_import >= 0) { /* Build the actual import arrays */ ierr = actual_arrays(zz, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, &actual_num_imp, &actual_imp_gids, &actual_imp_lids, &actual_imp_procs, &actual_imp_to_part, &actual_imp_allocated); if (ierr < 0) goto End; /* Compute communication map based on imports. */ msgtag = 32767; ierr = Zoltan_Comm_Create(&imp_plan, actual_num_imp, actual_imp_procs, zz->Communicator, msgtag, &actual_num_exp); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create."); goto End; } /* Compute actual export lists for packing objects */ if (actual_num_exp > 0) { actual_exp_allocated = 1; actual_exp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_exp); actual_exp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, actual_num_exp); actual_exp_procs = (int *) ZOLTAN_MALLOC(sizeof(int) * actual_num_exp); if (include_parts) actual_exp_to_part = (int *) ZOLTAN_MALLOC(sizeof(int)*actual_num_exp); if (actual_exp_gids == NULL || (num_lid_entries && actual_exp_lids == NULL) || actual_exp_procs == NULL || (import_to_part != NULL && actual_exp_to_part == NULL)) { Zoltan_Multifree(__FILE__, __LINE__, 4, &actual_exp_gids, &actual_exp_lids, &actual_exp_procs, &actual_exp_to_part); ierr = ZOLTAN_MEMERR; goto End; } } msgtag2 = 32766; ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_gids, (int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)), (char *) actual_exp_gids); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } if (num_lid_entries) { msgtag2--; ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_lids, (int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries), (char *) actual_exp_lids); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } } Zoltan_Comm_Info(imp_plan, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, actual_exp_procs, NULL); if (include_parts) { msgtag2--; ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_to_part, (int) sizeof(int), (char *) actual_exp_to_part); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } } /* Create inverse plan (i.e., plan based on exports) so can set * variable sizes. * (Zoltan_Comm_Do_Reverse(imp_plan, ...) allows sending variable * but does not tell how large to allocate receive buffer. */ ierr = Zoltan_Comm_Invert_Plan(&imp_plan); exp_plan = imp_plan; imp_plan = NULL; } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Import or export lists needed."); ierr = ZOLTAN_FATAL; goto End; } if (zz->Migrate.Pre_Migrate_PP != NULL) { zz->Migrate.Pre_Migrate_PP(zz->Migrate.Pre_Migrate_PP_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PRE_MIGRATE_PP_FN function."); goto End; } } if (zz->Migrate.Pre_Migrate != NULL) { zz->Migrate.Pre_Migrate(zz->Migrate.Pre_Migrate_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, num_export, export_global_ids, export_local_ids, export_procs, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PRE_MIGRATE_FN function."); goto End; } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done pre-migration processing"); id_size = Zoltan_Align(num_gid_entries * sizeof(ZOLTAN_ID_TYPE)); /* Note that alignment is not strictly necessary when ZOLTAN_ID_TYPE is int or unsigned int. */ aligned_int = Zoltan_Align(sizeof(int)); tag_size = id_size + aligned_int; /* * For each object, allow space for its global ID and its data plus * one int (for the object data size). * Zoltan will pack the global IDs; the application must pack the data * through the pack routine. Zoltan needs the global IDs for unpacking, * as the order of the data received during communication is not * necessarily the same order as import_global_ids[]. * Zoltan also needs to communicate the sizes of the objects because * only the sender knows the size of each object. */ if (actual_num_exp > 0) { sizes = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int)); if (!sizes) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } if (zz->Get_Obj_Size_Multi != NULL) { zz->Get_Obj_Size_Multi(zz->Get_Obj_Size_Multi_Data, num_gid_entries, num_lid_entries, actual_num_exp, actual_exp_gids, actual_exp_lids, sizes, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_OBJ_SIZE_MULTI function."); goto End; } } else { for (i = 0; i < actual_num_exp; i++) { lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL); sizes[i] = zz->Get_Obj_Size(zz->Get_Obj_Size_Data, num_gid_entries, num_lid_entries, &(actual_exp_gids[i*num_gid_entries]), lid, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_OBJ_SIZE function."); goto End; } } } total_send_size = 0; for (i = 0; i < actual_num_exp; i++) { sizes[i] = Zoltan_Align(sizes[i]); total_send_size += sizes[i] + tag_size; } export_buf = (char *) ZOLTAN_CALLOC(total_send_size, sizeof(char)); if (!export_buf) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } if (zz->Pack_Obj_Multi != NULL) { /* Allocate an index array for ZOLTAN_PACK_OBJ_MULTI_FN. */ idx = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int)); if (!idx) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } } /* * Pack the objects for export. */ idx_cnt = 0; tmp = export_buf; for (i = 0; i < actual_num_exp; i++) { /* Pack the object's global ID */ tmp_id = (ZOLTAN_ID_PTR) tmp; ZOLTAN_SET_GID(zz, tmp_id, &(actual_exp_gids[i*num_gid_entries])); tmp += id_size; /* Pack the object's size */ *((int *)tmp) = sizes[i]; tmp += aligned_int; /* If using ZOLTAN_PACK_OBJ_MULTI_FN, build the index array. */ idx_cnt += tag_size; if (idx != NULL) { idx[i] = idx_cnt; } tmp += sizes[i]; idx_cnt += sizes[i]; } if (zz->Pack_Obj_Multi != NULL) { if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Packing objects with multi-pack\n", zz->Proc, yo); } zz->Pack_Obj_Multi(zz->Pack_Obj_Multi_Data, num_gid_entries, num_lid_entries, actual_num_exp, actual_exp_gids, actual_exp_lids, (actual_exp_to_part!=NULL ? actual_exp_to_part : actual_exp_procs), sizes, idx, export_buf, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PACK_OBJ_MULTI function."); goto End; } } else { tmp = export_buf + tag_size; for (i = 0; i < actual_num_exp; i++) { if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Packing object with gid ", zz->Proc, yo); ZOLTAN_PRINT_GID(zz, &(actual_exp_gids[i*num_gid_entries])); printf("size = %d bytes\n", sizes[i]); } /* Pack the object's data */ lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL); dest = (actual_exp_to_part != NULL ? actual_exp_to_part[i] : actual_exp_procs[i]); zz->Pack_Obj(zz->Pack_Obj_Data, num_gid_entries, num_lid_entries, &(actual_exp_gids[i*num_gid_entries]), lid, dest, sizes[i], tmp, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PACK_OBJ function."); goto End; } tmp += sizes[i] + tag_size; } } ZOLTAN_FREE(&idx); tmp_id = NULL; } ZOLTAN_TRACE_DETAIL(zz, yo, "Done packing objects"); /* Modify sizes[] to contain message sizes, not object sizes */ for (i=0; i<actual_num_exp; i++) { sizes[i] += tag_size; } msgtag--; ierr = Zoltan_Comm_Resize(exp_plan, sizes, msgtag, &total_recv_size); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Resize."); goto End; } if (actual_num_imp > 0) { import_buf = (char *) ZOLTAN_MALLOC(total_recv_size); if (!import_buf) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } } /* * Send the export data using the communication plan. */ msgtag2 = 32765; ierr = Zoltan_Comm_Do(exp_plan, msgtag2, export_buf, 1, import_buf); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } /* * Free whatever memory we can. */ Zoltan_Comm_Destroy(&exp_plan); ZOLTAN_FREE(&export_buf); ZOLTAN_FREE(&sizes); ZOLTAN_TRACE_DETAIL(zz, yo, "Done communication"); /* * Perform application-specified processing before unpacking the data. */ if (zz->Migrate.Mid_Migrate_PP != NULL) { zz->Migrate.Mid_Migrate_PP(zz->Migrate.Mid_Migrate_PP_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_MID_MIGRATE_PP_FN function."); goto End; } } if (zz->Migrate.Mid_Migrate != NULL) { zz->Migrate.Mid_Migrate(zz->Migrate.Mid_Migrate_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, num_export, export_global_ids, export_local_ids, export_procs, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_MID_MIGRATE_FN function."); goto End; } } /* * Unpack the object data. */ if (actual_num_imp > 0) { if (zz->Unpack_Obj_Multi != NULL) { /* Allocate and fill input arrays for Unpack_Obj_Multi. */ sizes = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int)); tmp_id = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_imp); idx = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int)); if (!sizes || !tmp_id || !idx) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } tmp = import_buf; idx_cnt = 0; for (i = 0; i < actual_num_imp; i++) { /* Unpack the object's global ID */ ZOLTAN_SET_GID(zz, &(tmp_id[i*num_gid_entries]), (ZOLTAN_ID_PTR) tmp); tmp += id_size; /* Unpack the object's size */ sizes[i] = *((int *)tmp); tmp += aligned_int; /* If using ZOLTAN_UNPACK_OBJ_MULTI_FN, build the index array. */ idx_cnt += tag_size; if (idx != NULL) { idx[i] = idx_cnt; } tmp += sizes[i]; idx_cnt += sizes[i]; } if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Unpacking objects with multi-fn\n", zz->Proc,yo); } zz->Unpack_Obj_Multi(zz->Unpack_Obj_Multi_Data, num_gid_entries, actual_num_imp, tmp_id, sizes, idx, import_buf, &ierr); ZOLTAN_FREE(&import_buf); ZOLTAN_FREE(&sizes); ZOLTAN_FREE(&tmp_id); ZOLTAN_FREE(&idx); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_UNPACK_OBJ_MULTI_FN."); goto End; } } else { tmp = import_buf; for (i = 0; i < actual_num_imp; i++) { tmp_size = *((int *)(tmp + id_size)); if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Unpacking object with gid ", zz->Proc, yo); ZOLTAN_PRINT_GID(zz, (ZOLTAN_ID_PTR)tmp); printf("size = %d bytes\n", tmp_size); } /* Unpack the object's data */ zz->Unpack_Obj(zz->Unpack_Obj_Data, num_gid_entries, (ZOLTAN_ID_PTR) tmp, tmp_size, tmp + tag_size, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_UNPACK_OBJ_FN."); goto End; } tmp += (tmp_size + tag_size); } ZOLTAN_FREE(&import_buf); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done unpacking objects"); if (zz->Migrate.Post_Migrate_PP != NULL) { zz->Migrate.Post_Migrate_PP(zz->Migrate.Post_Migrate_PP_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_POST_MIGRATE_PP_FN function."); goto End; } } if (zz->Migrate.Post_Migrate != NULL) { zz->Migrate.Post_Migrate(zz->Migrate.Post_Migrate_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, num_export, export_global_ids, export_local_ids, export_procs, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_POST_MIGRATE_FN function."); goto End; } } End: if (actual_exp_allocated) { Zoltan_Multifree(__FILE__, __LINE__, 4, &actual_exp_gids, &actual_exp_lids, &actual_exp_procs, &actual_exp_to_part); } if (actual_imp_allocated) { Zoltan_Multifree(__FILE__, __LINE__, 4, &actual_imp_gids, &actual_imp_lids, &actual_imp_procs, &actual_imp_to_part); } if (ierr < 0) { if (exp_plan) Zoltan_Comm_Destroy(&exp_plan); Zoltan_Multifree(__FILE__, __LINE__, 5, &import_buf, &tmp_id, &sizes, &idx, &export_buf); } ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
static int Zoltan_LB( ZZ *zz, int include_parts, /* Flag indicating whether to generate part informtion; 0 if called by Zoltan_LB_Balance, 1 if called by Zoltan_LB_Partition. */ int *changes, /* Set to zero or one depending on if Zoltan determines a new decomposition or not: zero - No changes to the decomposition were made by the load-balancing algorithm; migration is not needed. one - A new decomposition is suggested by the load-balancer; migration is needed to establish the new decomposition. */ int *num_gid_entries, /* The number of array entries in a global ID; set to be the max over all processors in zz->Communicator of the parameter Num_Global_ID_Entries. */ int *num_lid_entries, /* The number of array entries in a local ID; set to be the max over all processors in zz->Communicator of the parameter Num_Local_ID_Entries. */ int *num_import_objs, /* The number of non-local objects in the processor's new decomposition. */ ZOLTAN_ID_PTR *import_global_ids,/* Array of global IDs for non-local objects (i.e., objs to be imported) in the processor's new decomposition. */ ZOLTAN_ID_PTR *import_local_ids, /* Array of local IDs for non-local objects (i.e., objs to be imported) in the processor's new decomposition. */ int **import_procs, /* Array of processor IDs for processors currently owning non-local objects (i.e., objs to be imported) in this processor's new decomposition. */ int **import_to_part, /* Partition to which the objects should be imported. */ int *num_export_objs, /* The number of local objects that need to be exported from the processor to establish the new decomposition. */ ZOLTAN_ID_PTR *export_global_ids,/* Array of global IDs for objects that need to be exported (assigned and sent to other processors) to establish the new decomposition. */ ZOLTAN_ID_PTR *export_local_ids, /* Array of local IDs for objects that need to be exported (assigned and sent to other processors) to establish the new decomposition. */ int **export_procs, /* Array of destination processor IDs for objects that need to be exported to establish the new decomposition. */ int **export_to_part /* Partition to which objects should be exported. */ ) { /* * Main load-balancing routine. * Input: a Zoltan structure with appropriate function pointers set. * Output: * changes * num_import_objs * import_global_ids * import_local_ids * import_procs * import_to_part * num_export_objs * export_global_ids * export_local_ids * export_procs * export_to_part * Return values: * Zoltan error code. */ char *yo = "Zoltan_LB"; int gmax; /* Maximum number of imported/exported objects over all processors. */ int error = ZOLTAN_OK; /* Error code */ double start_time, end_time; double lb_time[2] = {0.0,0.0}; char msg[256]; int comm[3],gcomm[3]; float *part_sizes = NULL, *fdummy = NULL; int wgt_dim, part_dim; int all_num_obj, i, ts, idIdx; struct Hash_Node **ht; int *export_all_procs, *export_all_to_part, *parts=NULL; ZOLTAN_ID_PTR all_global_ids=NULL, all_local_ids=NULL; ZOLTAN_ID_PTR gid; #ifdef ZOLTAN_OVIS struct OVIS_parameters ovisParameters; #endif ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS){ printf("Build configuration:\n"); Zoltan_Print_Configuration(" "); printf("\n"); Zoltan_Print_Key_Params(zz); } start_time = Zoltan_Time(zz->Timer); #ifdef ZOLTAN_OVIS Zoltan_OVIS_Setup(zz, &ovisParameters); if (zz->Proc == 0) printf("OVIS PARAMETERS %s %s %d %f\n", ovisParameters.hello, ovisParameters.dll, ovisParameters.outputLevel, ovisParameters.minVersion); ovis_enabled(zz->Proc, ovisParameters.dll); #endif /* * Compute Max number of array entries per ID over all processors. * Compute Max number of return arguments for Zoltan_LB_Balance. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; comm[2] = zz->LB.Return_Lists; MPI_Allreduce(comm, gcomm, 3, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = *num_gid_entries = gcomm[0]; zz->Num_LID = *num_lid_entries = gcomm[1]; zz->LB.Return_Lists = gcomm[2]; /* assume no changes */ *changes = 0; *num_import_objs = *num_export_objs = 0; *import_global_ids = NULL; *import_local_ids = NULL; *import_procs = NULL; *import_to_part = NULL; *export_global_ids = NULL; *export_local_ids = NULL; *export_procs = NULL; *export_to_part = NULL; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) goto End; if (zz->LB.Method == NONE) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) printf("%s Balancing method selected == NONE; no balancing performed\n", yo); error = ZOLTAN_WARN; goto End; } /* * Sync the random number generator across processors. */ Zoltan_Srand_Sync(Zoltan_Rand(NULL), NULL, zz->Communicator); /* Since generating a new partition, need to free old mapping vector */ zz->LB.OldRemap = zz->LB.Remap; zz->LB.Remap = NULL; error = Zoltan_LB_Build_PartDist(zz); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) goto End; if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { int i, np, fp; for (i = 0; i < zz->Num_Proc; i++) { Zoltan_LB_Proc_To_Part(zz, i, &np, &fp); printf("%d Proc_To_Part Proc %d NParts %d FPart %d\n", zz->Proc, i, np, fp); } } /* * Generate parts sizes. */ #ifdef ZOLTAN_OVIS /* set part sizes computed by OVIS, if requested. Processes set only their own value */ { float part_sizes[1]; int part_ids[1], wgt_idx[1]; wgt_idx[0] = 0; part_ids[0] = 0; ovis_getPartsize(&(part_sizes[0])); printf("Rank %d ps %f\n",zz->Proc, part_sizes[0]); /* clear out old part size info first */ Zoltan_LB_Set_Part_Sizes(zz, 0, -1, NULL, NULL, NULL); Zoltan_LB_Set_Part_Sizes(zz, 0, 1, part_ids, wgt_idx, part_sizes); } #endif wgt_dim = zz->Obj_Weight_Dim; part_dim = ((wgt_dim > 0) ? wgt_dim : 1); part_sizes = (float *) ZOLTAN_MALLOC(sizeof(float) * part_dim * zz->LB.Num_Global_Parts); if (part_sizes == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); error = ZOLTAN_MEMERR; goto End; } /* Get part sizes. */ Zoltan_LB_Get_Part_Sizes(zz, zz->LB.Num_Global_Parts, part_dim, part_sizes); #ifdef ZOLTAN_OVIS /* if (ovisParameters.outputlevel > 3) */ { int myRank = zz->Proc; if (myRank == 0){ int i, j; for (i = 0; i < zz->LB.Num_Global_Parts; i++){ for (j = 0; j < part_dim; j++){ printf("Rank %d AG: part_sizes[%d] = %f (Num_Global_Parts = %d, part_dim = %d)\n",zz->Proc, (i*part_dim+j), part_sizes[i*part_dim+j],zz->LB.Num_Global_Parts, part_dim); } } } } #endif /* * Call the actual load-balancing function. */ error = zz->LB.LB_Fn(zz, part_sizes, num_import_objs, import_global_ids, import_local_ids, import_procs, import_to_part, num_export_objs, export_global_ids, export_local_ids, export_procs, export_to_part); ZOLTAN_FREE(&part_sizes); if (error == ZOLTAN_FATAL || error == ZOLTAN_MEMERR){ sprintf(msg, "Partitioning routine returned code %d.", error); #ifdef HOST_LINUX if ((error == ZOLTAN_MEMERR) && (Zoltan_Memory_Get_Debug() > 0)){ Zoltan_write_linux_meminfo(0, "State of /proc/meminfo after malloc failure\n", 0); } #endif ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } else if (error){ if (zz->Debug_Level >ZOLTAN_DEBUG_NONE) { sprintf(msg, "Partitioning routine returned code %d.", error); ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done partitioning"); if (*num_import_objs >= 0) MPI_Allreduce(num_import_objs, &gmax, 1, MPI_INT, MPI_MAX, zz->Communicator); else /* use export data */ MPI_Allreduce(num_export_objs, &gmax, 1, MPI_INT, MPI_MAX, zz->Communicator); if (gmax == 0) { /* * Decomposition was not changed by the load balancing; no migration * is needed. */ if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) printf("%s No changes to the decomposition due to partitioning; " "no migration is needed.\n", yo); /* * Reset num_import_objs and num_export_objs; don't want to return * -1 for the arrays that weren't returned by ZOLTAN_LB_FN. */ *num_import_objs = *num_export_objs = 0; if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS){ /* * This parameter setting requires that all local objects * and their assignments appear in the export list. */ error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs, export_global_ids, export_local_ids, wgt_dim, &fdummy, export_to_part); if (error == ZOLTAN_OK){ ZOLTAN_FREE(&fdummy); if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs, ZOLTAN_SPECIAL_MALLOC_INT)){ for (i=0; i<*num_export_objs; i++) (*export_procs)[i] = zz->Proc; } else{ error = ZOLTAN_MEMERR; } } } goto End; } /* * Check whether we know the import data, export data, or both. * * If we were given the import data, * we know what the new decomposition should look like on the * processor, but we don't know which of our local objects we have * to export to other processors to establish the new decomposition. * Reverse the argument if we were given the export data. * * Unless we were given both maps, compute the inverse map. */ if (zz->LB.Return_Lists == ZOLTAN_LB_NO_LISTS) { if (*num_import_objs >= 0) Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids, import_procs, import_to_part); if (*num_export_objs >= 0) Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, export_procs, export_to_part); *num_import_objs = *num_export_objs = -1; } if (*num_import_objs >= 0){ if (*num_export_objs >= 0) { /* Both maps already available; nothing to do. */; } else if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) { /* Export lists are requested; compute export map */ error = Zoltan_Invert_Lists(zz, *num_import_objs, *import_global_ids, *import_local_ids, *import_procs, *import_to_part, num_export_objs, export_global_ids, export_local_ids, export_procs, export_to_part); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) { sprintf(msg, "Error building return arguments; " "%d returned by Zoltan_Compute_Destinations\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } if (zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) { /* Method returned import lists, but only export lists were desired. */ /* Import lists not needed; free them. */ *num_import_objs = -1; Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids, import_procs, import_to_part); } } } else { /* (*num_import_objs < 0) */ if (*num_export_objs >= 0) { /* Only export lists have been returned. */ if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) { /* Compute import map */ error = Zoltan_Invert_Lists(zz, *num_export_objs, *export_global_ids, *export_local_ids, *export_procs, *export_to_part, num_import_objs, import_global_ids, import_local_ids, import_procs, import_to_part); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) { sprintf(msg, "Error building return arguments; " "%d returned by Zoltan_Compute_Destinations\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } if (zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) { /* Method returned export lists, but only import lists are desired. */ /* Export lists not needed; free them. */ *num_export_objs = -1; Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, export_procs, export_to_part); } } } else { /* *num_export_objs < 0 && *num_import_objs < 0) */ if (zz->LB.Return_Lists) { /* No map at all available */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Load-balancing function returned " "neither import nor export data."); error = ZOLTAN_WARN; goto End; } } } if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) { /* * Normally, Zoltan_LB returns in the export lists all local * objects that are moving off processor, or that are assigned * to a part on the local processor that is not the * default part. This setting of Return_Lists requests * that all local objects be included in the export list. */ if (*num_export_objs == 0){ /* all local objects are remaining on processor */ error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs, export_global_ids, export_local_ids, wgt_dim, &fdummy, export_to_part); if (error == ZOLTAN_OK){ ZOLTAN_FREE(&fdummy); if (*num_export_objs) { if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs, ZOLTAN_SPECIAL_MALLOC_INT)){ for (i=0; i<*num_export_objs; i++) (*export_procs)[i] = zz->Proc; } else{ error = ZOLTAN_MEMERR; } } } if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)) goto End; } else{ all_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &error); if (*num_export_objs < all_num_obj){ /* Create a lookup table for exported IDs */ ts = Zoltan_Recommended_Hash_Size(*num_export_objs); ht = create_hash_table(zz, *export_global_ids, *num_export_objs, ts); /* Create a list of all gids, lids and parts */ error= Zoltan_Get_Obj_List_Special_Malloc(zz, &all_num_obj, &all_global_ids, &all_local_ids, wgt_dim, &fdummy, &parts); if ((error == ZOLTAN_OK) || (error == ZOLTAN_WARN)){ ZOLTAN_FREE(&fdummy); if ((Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_procs, all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0) || (Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_to_part, all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0)){ error = ZOLTAN_MEMERR; } } if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)){ sprintf(msg, "Error building complete export list; " "%d returned by Zoltan_Get_Obj_List\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } gid = all_global_ids; for (i=0; i < all_num_obj; i++, gid += zz->Num_GID){ idIdx = search_hash_table(zz, gid, ht, ts); if (idIdx >= 0){ export_all_procs[i] = (*export_procs)[idIdx]; export_all_to_part[i] = (*export_to_part)[idIdx]; } else{ export_all_procs[i] = zz->Proc; export_all_to_part[i] = parts[i]; } } free_hash_table(ht, ts); Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, export_procs, export_to_part); Zoltan_Special_Free(zz, (void **)(void*)&parts, ZOLTAN_SPECIAL_MALLOC_INT); *export_global_ids = all_global_ids; *export_local_ids = all_local_ids; *export_procs = export_all_procs; *export_to_part = export_all_to_part; *num_export_objs = all_num_obj; } } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done building return arguments"); end_time = Zoltan_Time(zz->Timer); lb_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: Objects to be imported to Proc %d\n", zz->Proc); for (i = 0; i < *num_import_objs; i++) { printf(" Obj: "); ZOLTAN_PRINT_GID(zz, &((*import_global_ids)[i*zz->Num_GID])); printf(" To part: %4d", (*import_to_part != NULL ? (*import_to_part)[i] : zz->Proc)); printf(" From processor: %4d\n", (*import_procs)[i]); } printf("\n"); printf("ZOLTAN: Objects to be exported from Proc %d\n", zz->Proc); for (i = 0; i < *num_export_objs; i++) { printf(" Obj: "); ZOLTAN_PRINT_GID(zz, &((*export_global_ids)[i*zz->Num_GID])); printf(" To part: %4d", (*export_to_part != NULL ? (*export_to_part)[i] : (*export_procs)[i])); printf(" To processor: %4d\n", (*export_procs)[i]); } Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* * If the Help_Migrate flag is set, perform migration for the application. */ if (zz->Migrate.Auto_Migrate) { ZOLTAN_TRACE_DETAIL(zz, yo, "Begin auto-migration"); start_time = Zoltan_Time(zz->Timer); error = Zoltan_Migrate(zz, *num_import_objs, *import_global_ids, *import_local_ids, *import_procs, *import_to_part, *num_export_objs, *export_global_ids, *export_local_ids, *export_procs, *export_to_part); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) { sprintf(msg, "Error in auto-migration; %d returned from " "Zoltan_Help_Migrate\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } end_time = Zoltan_Time(zz->Timer); lb_time[1] = end_time - start_time; ZOLTAN_TRACE_DETAIL(zz, yo, "Done auto-migration"); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[0], "ZOLTAN Partition: "); if (zz->Migrate.Auto_Migrate) Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[1], "ZOLTAN Migrate: "); } *changes = 1; End: ZOLTAN_TRACE_EXIT(zz, yo); return (error); }
/* * void malloc_new_objects(); * * gets the tags being imported into this processor, and sets up the * import_tags array, and the nrectags array. */ static int malloc_new_objects(ZZ *zz, int nsentags, pRegion exported_tags, ZOLTAN_ID_PTR exported_gids, ZOLTAN_ID_PTR exported_lids, int *tag_pids, int *nstags, pRegion *ex_tags, pRegion prev_tags, ZOLTAN_ID_PTR prev_gids, ZOLTAN_ID_PTR prev_lids, int npimtags, float *c3) { char *yo = "malloc_new_objects"; int i; /* index counter */ int nreceives; /* number of messages received */ pRegion t_b_exp; /* array of tags to be exported */ pRegion tmp = NULL; ZOLTAN_ID_PTR tmp_gids = NULL; ZOLTAN_ID_PTR tmp_lids = NULL; int msgtag, msgtag2; int j; int ierr = ZOLTAN_OK; int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; float im_load; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */ im_load = 0; msgtag = 32767; ierr = Zoltan_Comm_Create(&comm_plan, nsentags, tag_pids, zz->Communicator, msgtag, &nreceives); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Create."); ZOLTAN_TRACE_EXIT(zz, yo); return(ierr); } if (nreceives > 0) { tmp = (pRegion) ZOLTAN_MALLOC(nreceives * sizeof(Region)); tmp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, nreceives); tmp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, nreceives); if(tmp == NULL || !tmp_gids || (num_lid_entries && !tmp_lids)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } } msgtag2 = 32766; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_tags, sizeof(Region), (char *) tmp); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return(ierr); } msgtag2--; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_gids, sizeof(ZOLTAN_ID_TYPE)*num_gid_entries, (char *) tmp_gids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); fprintf(stderr, "OCT %s Error %s returned from Zoltan_Comm_Do\n", yo, (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return(ierr); } if (num_lid_entries > 0) { msgtag2--; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_lids, sizeof(ZOLTAN_ID_TYPE)*num_lid_entries, (char *) tmp_lids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return(ierr); } } ierr = Zoltan_Comm_Destroy(&comm_plan); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Destroy."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&tmp); return(ierr); } /* get each message sent, and store region in import array */ j=0; for (i=0; i<nreceives; i++) { im_load += tmp[i].Weight; if(tmp[i].newProc != zz->Proc) { j++; } } if((j + npimtags) != 0) { /* malloc import array */ if((t_b_exp = (pRegion)ZOLTAN_MALLOC((j+npimtags)*sizeof(Region)))==NULL){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return ZOLTAN_MEMERR; } } else t_b_exp = NULL; /* setup return pointer */ (*ex_tags) = t_b_exp; j=0; for (i=0; i<nreceives; i++) { if(tmp[i].newProc != zz->Proc) { t_b_exp[j] = tmp[i]; t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz); t_b_exp[j].Local_ID = ZOLTAN_MALLOC_LID(zz); if (!(t_b_exp[j].Global_ID) || (num_lid_entries && !(t_b_exp[j].Local_ID))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return ZOLTAN_MEMERR; } ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID, &(tmp_gids[i*num_gid_entries])); ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID, &(tmp_lids[i*num_lid_entries])); j++; } } if(npimtags > 0) { for(i=0; i<npimtags; i++) { t_b_exp[j] = prev_tags[i]; t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz); t_b_exp[j].Local_ID = ZOLTAN_MALLOC_LID(zz); if (!(t_b_exp[j].Global_ID) || (num_lid_entries && !(t_b_exp[j].Local_ID))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); return ZOLTAN_MEMERR; } ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID, &(prev_gids[i*num_gid_entries])); ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID, &(prev_lids[i*num_lid_entries])); j++; } } *nstags = j; ZOLTAN_FREE(&tmp); ZOLTAN_FREE(&tmp_gids); ZOLTAN_FREE(&tmp_lids); if((*nstags == 0) && (*ex_tags != NULL)) { ZOLTAN_TRACE_DETAIL(zz, yo, "Fatal error, import tags not empty but no tags received\n"); return ZOLTAN_FATAL; } *c3 = im_load; return ierr; }
/* * void tag_regions() * Iterates through the list of octants on the processor and finds which * are to be migrated. It then looks at the region list for those octants * and stores the migrating regions into the export_tags array. */ static int tag_regions(ZZ *zz, pOctant *octs, int *newpids, int nocts, Region **exported_tags, ZOLTAN_ID_PTR *exported_gids, ZOLTAN_ID_PTR *exported_lids, int *nsentags, int **tag_pids, Region **p_tags, ZOLTAN_ID_PTR *p_gids, ZOLTAN_ID_PTR *p_lids, int *npimtags, float *c2, int *max_objs) { char *yo = "tag_regions"; int i; /* index counter */ pRegion regionlist; /* list of region on this processor */ int index; /* index counter */ int index2; /* yet another index counter */ int count; /* count of objects exported form this processor */ int count2; /* count of objects that are kept on processor */ int count3; int *exported_pids; /* array of pids where regions are being exported to */ pRegion mtags; /* object tags of objects to be migrated */ pRegion ptags; /* tags of objects that were previously migrated */ float ex_load; int ierr = ZOLTAN_OK; int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; ex_load = 0; (*max_objs) = 0; if (!nsentags) return ierr; /* find how many objects have been exported */ count = 0; /* find number of local objs to export */ count2 = 0; count3 = 0; for (i=0; i<nocts; i++) { if(Zoltan_Oct_isTerminal(octs[i])) { (*max_objs) += Zoltan_Oct_nRegions(octs[i]); regionlist = Zoltan_Oct_regionlist(octs[i]); while(regionlist != NULL) { count3++; if(regionlist->Proc != zz->Proc) { count++; if(newpids[i] != zz->Proc) regionlist->newProc = newpids[i]; else regionlist->newProc = zz->Proc; } else { if(newpids[i] != zz->Proc) { count2++; regionlist->newProc = newpids[i]; } else regionlist->newProc = zz->Proc; } regionlist = regionlist->next; /* get next region */ } } } #if 0 { { if (newpids[i]!=zz->Proc) { count+=Zoltan_Oct_nRegions(octs[i]); } else { pRegion regions; regions = Zoltan_Oct_regionlist(octs[i]); while(regions != NULL) { if(regions->Proc != zz->Proc) count2++; regions = regions->next; } } } } #endif /* set up the return pointers */ *nsentags = count; *npimtags = count2; if (!exported_tags) { return ierr; } if (count > 0) { /* allocate some space */ if((mtags=(pRegion)ZOLTAN_MALLOC((unsigned)count*sizeof(Region)))==NULL){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if((exported_pids = (int *)ZOLTAN_MALLOC((unsigned)count*sizeof(int))) == NULL){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&mtags); return ZOLTAN_MEMERR; } *exported_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, count); *exported_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, count); if(!(*exported_gids) || (num_lid_entries && !(*exported_lids))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&mtags); ZOLTAN_FREE(&exported_pids); return ZOLTAN_MEMERR; } } else { mtags = NULL; exported_pids = NULL; *exported_gids = NULL; *exported_lids = NULL; } /* set up return pointers */ *exported_tags=mtags; *tag_pids = exported_pids; if (count2 > 0) { /* allocate some space */ if((ptags=(pRegion)ZOLTAN_MALLOC((unsigned)count2*sizeof(Region)))==NULL){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient Memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&mtags); ZOLTAN_FREE(&exported_pids); ZOLTAN_FREE(exported_gids); ZOLTAN_FREE(exported_lids); return ZOLTAN_MEMERR; } *p_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, count2); *p_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, count2); if(!(*p_gids) || (num_lid_entries && !(*p_lids))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient Memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&mtags); ZOLTAN_FREE(&exported_pids); ZOLTAN_FREE(exported_gids); ZOLTAN_FREE(exported_lids); ZOLTAN_FREE(&ptags); ZOLTAN_FREE(p_gids); ZOLTAN_FREE(p_lids); return ZOLTAN_MEMERR; } } else { ptags = NULL; *p_gids = NULL; *p_lids = NULL; } /* set up return pointers */ *p_tags=ptags; index = index2 = 0; for (i=0; i<nocts; i++) { if(Zoltan_Oct_isTerminal(octs[i])) { regionlist = Zoltan_Oct_regionlist(octs[i]); while(regionlist != NULL) { if(regionlist->Proc != zz->Proc) { /* place information in the appropritate array */ mtags[index] = *regionlist; ZOLTAN_SET_GID(zz, &((*exported_gids)[index*num_gid_entries]), regionlist->Global_ID); ZOLTAN_SET_LID(zz, &((*exported_lids)[index*num_lid_entries]), regionlist->Local_ID); /*ex_load += (float)(regionlist->Weight);*/ exported_pids[index] = regionlist->Proc; index++; /* increment counter */ } else if(newpids[i] != zz->Proc) { ptags[index2] = *regionlist; /* get region information */ ZOLTAN_SET_GID(zz, &((*p_gids)[index2*num_gid_entries]), regionlist->Global_ID); ZOLTAN_SET_LID(zz, &((*p_lids)[index2*num_lid_entries]), regionlist->Local_ID); index2++; /* increment counter */ } regionlist = regionlist->next; /* get next region */ } } } if (index!=count) { /* error check */ ZOLTAN_TRACE_DETAIL(zz, yo, "Fatal error, inconsistent number of regions.\n"); return ZOLTAN_FATAL; } *c2 = ex_load; return ierr; }
int Zoltan_Order( ZZ *zz, /* Zoltan structure */ int *num_gid_entries, /* # of entries for a global id */ int *num_lid_entries, /* # of entries for a local id */ int num_obj, /* Number of objects to order */ ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */ /* The application must allocate enough space */ ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */ /* The application must allocate enough space */ int *rank, /* rank[i] is the rank of gids[i] */ int *iperm, /* inverse permutation of rank */ ZOS *order_info /* Method-specific ordering info. Currently not used. */ ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids or enough space to store such a list * lids, a list of local ids or enough space to store such a list * Output: * num_gid_entries * num_lid_entries * gids, a list of global ids (filled in if empty on entry) * lids, a list of local ids (filled in if empty on entry) * rank, rank[i] is the global rank of gids[i] * iperm, inverse permutation of rank * order_info, a Zoltan Ordering Struct with additional info. * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; int *vtxdist; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = *num_gid_entries = gcomm[0]; zz->Num_LID = *num_lid_entries = gcomm[1]; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); strncpy(opt.order_type, "GLOBAL", MAX_PARAM_STRING_LEN); opt.use_order_info = 0; opt.start_index = 0; opt.reorder = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type); Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index); Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); if (opt.use_order_info == 0) order_info = NULL; /* * Check that the user has allocated space for the return args. */ if (!(gids && lids && rank && iperm)){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } else if (!strcmp(opt.method, "NODEND")) { Order_fn = Zoltan_ParMetis_Order; } else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "GLOBAL"); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL){ ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /* * Call the actual ordering function. */ ierr = (*Order_fn)(zz, num_obj, gids, lids, rank, iperm, &opt, order_info); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN){ ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* Compute inverse permutation if necessary */ ierr = Zoltan_Get_Distribution(zz, &vtxdist); if (ierr){ /* Error */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n"); return (ierr); } if (!(opt.return_args & RETURN_RANK)){ /* Compute rank from iperm */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, iperm, rank, vtxdist, opt.order_type, opt.start_index); } else if (!(opt.return_args & RETURN_IPERM)){ /* Compute iperm from rank */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, rank, iperm, vtxdist, opt.order_type, opt.start_index); } ZOLTAN_FREE(&vtxdist); ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i, nobjs; nobjs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &i); Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < nobjs; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(*num_gid_entries)])); printf(", rank = %3d\n", rank[i]); } printf("\n"); printf("ZOLTAN: inverse permutation on Proc %d\n", zz->Proc); for (i = 0; i < nobjs; i++) { printf("iperm[%3d] = %3d\n", i, iperm[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } ZOLTAN_TRACE_EXIT(zz, yo); if (ierr) return (ierr); else return (ZOLTAN_OK); }
int Zoltan_Invert_Lists( ZZ *zz, /* Zoltan structure. */ int num_in, /* Number of objects in the input lists. */ ZOLTAN_ID_PTR in_global_ids, /* Array of input global IDs. */ ZOLTAN_ID_PTR in_local_ids, /* Array of input local IDs. */ int *in_procs, /* Array of processor IDs of processors owning the input objects. */ int *in_to_part, /* Optional: Array of partition numbers to which input objects should be assigned. */ int *num_out, /* Returned value: Number of output objs. */ ZOLTAN_ID_PTR *out_global_ids,/* Returned value: Array of global IDs of output objects. */ ZOLTAN_ID_PTR *out_local_ids, /* Returned value: Array of local IDs of output objects. */ int **out_procs, /* Returned value: Array of processor IDs to which output objects are assigned. */ int **out_to_part /* Optional: Returned value: Array of partition numbers to which output objects should be assigned. */ ) { /* * Routine to compute the inverse map. Can be used in two ways: * 1. Given, for each processor, a list of objects to be received by the * processor, compute the list of objects that the processor needs to send * to other processors to satisfy their needs. * 2. Given, for each processor, a list of objects to be sent to other * processors, compute the list of objects that the processor needs to receive * to satisfy its needs. */ char *yo = "Zoltan_Invert_Lists"; char msg[256]; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned communication routines */ int msgtag, msgtag2; /* Message tags for communication routines */ int num_gid_entries, num_lid_entries; /* Length of global and local ids */ int include_parts; /* Flag indicating whether to compute inverse list for partitions. */ int ierr, ret_ierr = ZOLTAN_OK; ZOLTAN_TRACE_ENTER(zz, yo); /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Check that all procs use the same id types. */ ierr = check_invert_input(zz, num_in, in_procs, in_to_part, &num_gid_entries, &num_lid_entries, &include_parts); if (ierr != ZOLTAN_OK) { ZOLTAN_TRACE_EXIT(zz, yo); return ierr; } /* Initialize returned arrays. */ *out_global_ids = NULL; *out_local_ids = NULL; *out_procs = NULL; if (include_parts) *out_to_part = NULL; /* * Compute communication map and num_out, the number of objs this * processor has to out to establish the new decomposition. */ msgtag = 32767; ierr = Zoltan_Comm_Create(&comm_plan, num_in, in_procs, zz->Communicator, msgtag, num_out); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Create.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; goto End; } ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm create"); /* * Allocate space for the object tags that need to be outed. Communicate * to get the list of objects to be outed. */ if (*num_out > 0) { if (!Zoltan_Special_Malloc(zz,(void **)out_global_ids,*num_out, ZOLTAN_SPECIAL_MALLOC_GID)) { ret_ierr = ZOLTAN_MEMERR; goto End; } if (!Zoltan_Special_Malloc(zz,(void **)out_local_ids,*num_out, ZOLTAN_SPECIAL_MALLOC_LID)) { ret_ierr = ZOLTAN_MEMERR; goto End; } if (!Zoltan_Special_Malloc(zz,(void **)out_procs,*num_out, ZOLTAN_SPECIAL_MALLOC_INT)) { ret_ierr = ZOLTAN_MEMERR; goto End; } if (include_parts) { if (!Zoltan_Special_Malloc(zz,(void **)out_to_part,*num_out, ZOLTAN_SPECIAL_MALLOC_INT)) { ret_ierr = ZOLTAN_MEMERR; goto End; } } } /* * Use the communication plan to send global IDs, local IDs, and processor * numbers. Do in separate communications to avoid a memory copy and to * simplify implementation when a data type is added to the comm. package * (to support heterogeneous computing). */ msgtag2 = 32766; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_global_ids, (int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)), (char *) *out_global_ids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; } if (num_lid_entries) { msgtag2--; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_local_ids, (int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries), (char *) *out_local_ids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; } } Zoltan_Comm_Info(comm_plan, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, *out_procs, NULL); if (include_parts) { msgtag2--; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_to_part, (int) sizeof(int), (char *) *out_to_part); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm_do"); End: Zoltan_Comm_Destroy(&comm_plan); if (ret_ierr == ZOLTAN_MEMERR) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); Zoltan_Special_Free(zz,(void**)out_global_ids,ZOLTAN_SPECIAL_MALLOC_GID); Zoltan_Special_Free(zz,(void**)out_local_ids,ZOLTAN_SPECIAL_MALLOC_LID); Zoltan_Special_Free(zz,(void**)out_procs,ZOLTAN_SPECIAL_MALLOC_INT); if (include_parts) Zoltan_Special_Free(zz,(void**)out_to_part,ZOLTAN_SPECIAL_MALLOC_INT); } ZOLTAN_TRACE_EXIT(zz, yo); return (ret_ierr); }
int Zoltan_HG_Create_Mirror ( ZZ *zz, HGraph *hg ) { int inlength, outlength; /* input/output array lengths */ int *index, *data; /* pointers to input information */ int *outindex, *outdata; char *yo = "Zoltan_HG_Create_Mirror"; ZOLTAN_TRACE_ENTER(zz, yo); /* determine which data to "mirror" and set corresponding data pointers. */ if (hg && (hg->nEdge == 0 || hg->hindex) && (hg->nPins == 0 || hg->hvertex) && !hg->vindex && !hg->vedge) { ZOLTAN_TRACE_DETAIL(zz, yo, "Have hindex; building vindex."); inlength = hg->nEdge; outlength = hg->nVtx; index = hg->hindex; data = hg->hvertex; outindex = hg->vindex = (int*) ZOLTAN_MALLOC((hg->nVtx+1) * sizeof(int)); outdata = hg->vedge = (int*) ZOLTAN_MALLOC (hg->nPins * sizeof(int)); if (outindex == NULL || (hg->nPins > 0 && outdata == NULL)) { Zoltan_Multifree (__FILE__, __LINE__, 2, &hg->vindex, &hg->vedge); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } } else if (hg && (hg->nVtx == 0 || hg->vindex) && (hg->nPins == 0 || hg->vedge) && !hg->hindex && !hg->hvertex) { ZOLTAN_TRACE_DETAIL(zz, yo, "Have vindex; building hindex."); inlength = hg->nVtx; outlength = hg->nEdge; index = hg->vindex; data = hg->vedge; outindex = hg->hindex = (int*) ZOLTAN_MALLOC((hg->nEdge+1) * sizeof(int)); outdata = hg->hvertex = (int*) ZOLTAN_MALLOC(hg->nPins * sizeof(int)); if (outindex == NULL || (hg->nPins > 0 && outdata == NULL)) { Zoltan_Multifree (__FILE__, __LINE__, 2, &hg->hindex, &hg->hvertex); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } } else { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input error."); return ZOLTAN_FATAL; /* unable to proceed */ } Zoltan_HG_Mirror(inlength, index, data, outlength, outindex, outdata); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_OK; }
int Zoltan_ParMetis_Order( ZZ *zz, /* Zoltan structure */ int num_obj, /* Number of (local) objects to order. */ ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */ /* The application must allocate enough space */ ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */ /* The application must allocate enough space */ ZOLTAN_ID_PTR rank, /* rank[i] is the rank of gids[i] */ int *iperm, ZOOS *order_opt /* Ordering options, parsed by Zoltan_Order */ ) { static char *yo = "Zoltan_ParMetis_Order"; int i, n, ierr; ZOLTAN_Output_Order ord; ZOLTAN_Third_Graph gr; #ifdef ZOLTAN_PARMETIS MPI_Comm comm = zz->Communicator;/* don't want to risk letting external packages changing our communicator */ #endif indextype numflag = 0; int timer_p = 0; int get_times = 0; int use_timers = 0; double times[5]; ZOLTAN_ID_PTR l_gids = NULL; ZOLTAN_ID_PTR l_lids = NULL; indextype options[MAX_PARMETIS_OPTIONS]; char alg[MAX_PARAM_STRING_LEN]; ZOLTAN_TRACE_ENTER(zz, yo); #ifdef ZOLTAN_PARMETIS #if TPL_USE_DATATYPE != TPL_METIS_DATATYPES #ifdef TPL_FLOAT_WEIGHT i = 1; #else i = 0; #endif if ((sizeof(indextype) != sizeof(idxtype)) || (sizeof(weighttype) != sizeof(idxtype)) || i){ ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Not supported: Multiple 3rd party libraries with incompatible " "data types."); return ZOLTAN_FATAL; } #endif #endif memset(&gr, 0, sizeof(ZOLTAN_Third_Graph)); memset(&ord, 0, sizeof(ZOLTAN_Output_Order)); memset(times, 0, sizeof(times)); ord.order_opt = order_opt; if (!order_opt){ /* If for some reason order_opt is NULL, allocate a new ZOOS here. */ /* This should really never happen. */ order_opt = (ZOOS *) ZOLTAN_MALLOC(sizeof(ZOOS)); strcpy(order_opt->method,"PARMETIS"); } ierr = Zoltan_Parmetis_Parse(zz, options, alg, NULL, NULL, &ord); /* ParMetis only computes the rank vector */ order_opt->return_args = RETURN_RANK; /* Check that num_obj equals the number of objects on this proc. */ /* This constraint may be removed in the future. */ n = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN)){ /* Return error code */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Num_Obj returned error."); return(ZOLTAN_FATAL); } if (n != num_obj){ /* Currently this is a fatal error. */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input num_obj does not equal the " "number of objects."); return(ZOLTAN_FATAL); } /* Do not use weights for ordering */ gr.obj_wgt_dim = -1; gr.edge_wgt_dim = -1; gr.num_obj = num_obj; /* Check what ordering type is requested */ if (order_opt){ SET_GLOBAL_GRAPH(&gr.graph_type); /* GLOBAL by default */ #ifdef ZOLTAN_PARMETIS if ((strcmp(order_opt->method, "METIS") == 0)) #endif /* ZOLTAN_PARMETIS */ SET_LOCAL_GRAPH(&gr.graph_type); } gr.get_data = 1; if (IS_LOCAL_GRAPH(gr.graph_type) && zz->Num_Proc > 1) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Serial ordering on more than 1 process: " "set ParMetis instead."); return(ZOLTAN_FATAL); } timer_p = Zoltan_Preprocess_Timer(zz, &use_timers); /* Start timer */ get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME); if (get_times){ MPI_Barrier(zz->Communicator); times[0] = Zoltan_Time(zz->Timer); } ierr = Zoltan_Preprocess_Graph(zz, &l_gids, &l_lids, &gr, NULL, NULL, NULL); if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) { Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL); return (ierr); } /* Allocate space for separator sizes */ if (IS_GLOBAL_GRAPH(gr.graph_type)) { if (Zoltan_TPL_Order_Init_Tree(&zz->TPL_Order, 2*zz->Num_Proc, zz->Num_Proc) != ZOLTAN_OK) { /* Not enough memory */ Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord); ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } ord.sep_sizes = (indextype*)ZOLTAN_MALLOC((2*zz->Num_Proc+1)*sizeof(indextype)); if (ord.sep_sizes == NULL) { Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord); ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } memset(ord.sep_sizes, 0, (2*zz->Num_Proc+1)*sizeof(int)); /* It seems parmetis don't initialize correctly */ } /* Allocate space for direct perm */ ord.rank = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype)); if (!ord.rank){ /* Not enough memory */ Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord); ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } if (IS_LOCAL_GRAPH(gr.graph_type)){ /* Allocate space for inverse perm */ ord.iperm = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype)); if (!ord.iperm){ /* Not enough memory */ Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord); ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } } else ord.iperm = NULL; /* Get a time here */ if (get_times) times[1] = Zoltan_Time(zz->Timer); #ifdef ZOLTAN_PARMETIS if (IS_GLOBAL_GRAPH(gr.graph_type)){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library"); ParMETIS_V3_NodeND (gr.vtxdist, gr.xadj, gr.adjncy, &numflag, options, ord.rank, ord.sep_sizes, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else #endif /* ZOLTAN_PARMETIS */ #if defined(ZOLTAN_METIS) || defined(ZOLTAN_PARMETIS) if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Be careful : permutation parameters are in the opposite order */ indextype numobj = gr.num_obj; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library"); order_opt->return_args = RETURN_RANK|RETURN_IPERM; /* We provide directly all the permutations */ #if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5 options[0] = 0; /* Use default options for METIS. */ METIS_NodeND(&numobj, gr.xadj, gr.adjncy, &numflag, options, ord.iperm, ord.rank); #else METIS_SetDefaultOptions(options); METIS_NodeND(&numobj, gr.xadj, gr.adjncy, NULL, options, ord.iperm, ord.rank); /* NULL is vwgt -- new interface in v4 */ #endif ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library"); } #endif /* ZOLTAN_METIS */ /* Get a time here */ if (get_times) times[2] = Zoltan_Time(zz->Timer); if (IS_GLOBAL_GRAPH(gr.graph_type)){ /* Update Elimination tree */ int numbloc; int start; int leaf; int *converttab; int levelmax; levelmax = mylog2(zz->Num_Proc) + 1; converttab = (int*)ZOLTAN_MALLOC(zz->Num_Proc*2*sizeof(int)); memset(converttab, 0, zz->Num_Proc*2*sizeof(int)); /* Determine the first node in each separator, store it in zz->TPL_Order.start */ for (numbloc = 0, start=0, leaf=0; numbloc < zz->Num_Proc /2; numbloc++) { int father; father = zz->Num_Proc + numbloc; converttab[start] = 2*numbloc; zz->TPL_Order.leaves[leaf++]=start; zz->TPL_Order.ancestor[start] = start + 2; converttab[start+1] = 2*numbloc+1; zz->TPL_Order.leaves[leaf++]=start+1; zz->TPL_Order.ancestor[start+1] = start + 2; start+=2; do { converttab[start] = father; if (father %2 == 0) { int nextoffset; int level; level = mylog2(2*zz->Num_Proc - 1 - father); nextoffset = (1<<(levelmax-level)); zz->TPL_Order.ancestor[start] = start+nextoffset; start++; break; } else { zz->TPL_Order.ancestor[start] = start+1; start++; father = zz->Num_Proc + father/2; } } while (father < 2*zz->Num_Proc - 1); } zz->TPL_Order.start[0] = 0; zz->TPL_Order.ancestor [2*zz->Num_Proc - 2] = -1; for (numbloc = 1 ; numbloc < 2*zz->Num_Proc ; numbloc++) { int oldblock=converttab[numbloc-1]; zz->TPL_Order.start[numbloc] = zz->TPL_Order.start[numbloc-1] + ord.sep_sizes[oldblock]; } ZOLTAN_FREE(&converttab); ZOLTAN_FREE(&ord.sep_sizes); zz->TPL_Order.leaves[zz->Num_Proc] = -1; zz->TPL_Order.nbr_leaves = zz->Num_Proc; zz->TPL_Order.nbr_blocks = 2*zz->Num_Proc-1; } else { /* No tree */ zz->TPL_Order.nbr_blocks = 0; zz->TPL_Order.start = NULL; zz->TPL_Order.ancestor = NULL; zz->TPL_Order.leaves = NULL; } /* Correct because no redistribution */ memcpy(gids, l_gids, n*zz->Num_GID*sizeof(ZOLTAN_ID_TYPE)); memcpy(lids, l_lids, n*zz->Num_LID*sizeof(ZOLTAN_ID_TYPE)); ierr = Zoltan_Postprocess_Graph (zz, l_gids, l_lids, &gr, NULL, NULL, NULL, &ord, NULL); ZOLTAN_FREE(&l_gids); ZOLTAN_FREE(&l_lids); /* Get a time here */ if (get_times) times[3] = Zoltan_Time(zz->Timer); if (get_times) Zoltan_Third_DisplayTime(zz, times); if (use_timers) ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator); if (sizeof(indextype) == sizeof(ZOLTAN_ID_TYPE)){ memcpy(rank, ord.rank, gr.num_obj*sizeof(indextype)); } else{ for (i=0; i < gr.num_obj; i++){ rank[i] = (ZOLTAN_ID_TYPE)ord.rank[i]; } } if ((ord.iperm != NULL) && (iperm != NULL)){ if (sizeof(indextype) == sizeof(int)){ memcpy(iperm, ord.iperm, gr.num_obj*sizeof(indextype)); } else{ for (i=0; i < gr.num_obj; i++){ iperm[i] = (int)ord.iperm[i]; } } } if (ord.iperm != NULL) ZOLTAN_FREE(&ord.iperm); ZOLTAN_FREE(&ord.rank); /* Free all other "graph" stuff */ Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); }
int Zoltan_ParMetis( ZZ *zz, /* Zoltan structure */ float *part_sizes, /* Input: Array of size zz->Num_Global_Parts containing the percentage of work to be assigned to each partition. */ int *num_imp, /* number of objects to be imported */ ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */ ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */ int **imp_procs, /* list of processors to import from */ int **imp_to_part, /* list of partitions to which imported objects are assigned. */ int *num_exp, /* number of objects to be exported */ ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */ ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */ int **exp_procs, /* list of processors to export to */ int **exp_to_part /* list of partitions to which exported objects are assigned. */ ) { char *yo = "Zoltan_ParMetis"; int ierr; ZOLTAN_Third_Graph gr; ZOLTAN_Third_Geom *geo = NULL; ZOLTAN_Third_Vsize vsp; ZOLTAN_Third_Part prt; ZOLTAN_Output_Part part; ZOLTAN_ID_PTR global_ids = NULL; ZOLTAN_ID_PTR local_ids = NULL; int use_timers = 0; int timer_p = -1; int get_times = 0; double times[5]; double pmv3_itr = 0.0; realtype itr = 0.0; indextype options[MAX_PARMETIS_OPTIONS]; char alg[MAX_PARAM_STRING_LEN]; #ifdef ZOLTAN_PARMETIS MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */ /* change our zz struct. */ #endif indextype i; realtype *imb_tols; indextype ncon; indextype edgecut; indextype wgtflag; indextype numflag = 0; indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */ ZOLTAN_TRACE_ENTER(zz, yo); Zoltan_Third_Init(&gr, &prt, &vsp, &part, imp_gids, imp_lids, imp_procs, imp_to_part, exp_gids, exp_lids, exp_procs, exp_to_part); if (sizeof(realtype) != sizeof(float)) { int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1); prt.input_part_sizes = (realtype *) ZOLTAN_MALLOC(tmp * sizeof(realtype)); for (i = 0; i < tmp; i++) prt.input_part_sizes[i] = (realtype) part_sizes[i]; /* KDD 2/2014: removed re-scaling part sizes so they sum to one. * part_sizes are already scaled in Zoltan_LB_Get_Part_Sizes. * plus, the code here was wrong for multiple object weights. * similar scaling code did not exist in the Scotch interface. */ prt.part_sizes = prt.input_part_sizes; } else prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes; ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL); if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) { Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL); return (ierr); } gr.graph_type = 0; #ifdef ZOLTAN_PARMETIS SET_GLOBAL_GRAPH(&gr.graph_type); /* Select type of graph, negative because we impose them */ /* TODO: add a parameter to select the type, shared with Scotch */ /* if (strcmp (graph_type, "GLOBAL") != 0) { */ /* gr.graph_type = - LOCAL_GRAPH; */ /* if (zz->Num_Proc > 1) { */ /* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */ /* gr.graph_type = - GLOBAL_GRAPH; */ /* retval = ZOLTAN_WARN; */ /* } */ /* } */ #else /* graph is local */ SET_LOCAL_GRAPH(&gr.graph_type); #endif /* ZOLTAN_PARMETIS */ /* Some algorithms use geometry data */ if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */ geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom)); memset (geo, 0, sizeof(ZOLTAN_Third_Geom)); /* ParMETIS will crash if geometric method and some procs have no nodes. */ /* Avoid fatal crash by setting scatter to level 2 or higher. */ gr.scatter_min = 2; if (geo == NULL) { ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory."); return (ZOLTAN_MEMERR); } if (strcmp(alg, "PARTGEOM") == 0) { gr.get_data = 0; } } timer_p = Zoltan_Preprocess_Timer(zz, &use_timers); /* Start timer */ get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME); if (get_times){ MPI_Barrier(zz->Communicator); times[0] = Zoltan_Time(zz->Timer); } vsp.vsize_malloc = 0; #ifdef PARMETIS31_ALWAYS_FREES_VSIZE if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) { /* ParMETIS will free this memory; use malloc to allocate so ZOLTAN_MALLOC counters don't show an error. */ vsp.vsize_malloc = 1 ; } #endif /* PARMETIS31_ALWAYS_FREES_VSIZE */ ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr, geo, &prt, &vsp); if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) { Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL); return (ierr); } /* Get object sizes if requested */ if (options[PMV3_OPT_USE_OBJ_SIZE] && (zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) && (!strcmp(alg, "ADAPTIVEREPART") || gr.final_output)) gr.showMoveVol = 1; /* Get a time here */ if (get_times) times[1] = Zoltan_Time(zz->Timer); /* Get ready to call ParMETIS */ edgecut = -1; wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0); numflag = 0; ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1); if (!prt.part_sizes){ ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL."); } if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) { for (i=0; i<num_part; i++){ indextype j; printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i); for (j=0; j<ncon; j++) printf("%f ", prt.part_sizes[i*ncon+j]); printf("\n"); } } /* if (strcmp(alg, "ADAPTIVEREPART") == 0) */ for (i = 0; i < num_part*ncon; i++) if (prt.part_sizes[i] == 0) ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! " "ParMETIS 3.x will likely fail. Please use a " "different method, or remove the zero-sized " "parts from the problem."); /* Set Imbalance Tolerance for each weight component. */ imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype)); if (!imb_tols){ /* Not enough memory */ ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } for (i=0; i<ncon; i++) imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]); /* Now we can call ParMetis */ /* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */ #ifdef ZOLTAN_PARMETIS if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */ /* First check for ParMetis 3 routines */ if (strcmp(alg, "PARTKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartKway"); ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "PARTGEOMKWAY") == 0){ indextype ndims = geo->ndims; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartGeomKway"); ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts, &wgtflag, &numflag, &ndims, geo->xyz, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "PARTGEOM") == 0){ indextype ndims = geo->ndims; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartGeom"); ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "ADAPTIVEREPART") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_AdaptiveRepart"); ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, &itr, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "REFINEKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_RefineKway"); ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else { /* Sanity check: This should never happen! */ char msg[256]; sprintf(msg, "Unknown ParMetis algorithm %s.", alg); ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg); } } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_METIS /* TODO: I don't know how to set balance ! */ if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Check for Metis routines */ if (strcmp(alg, "PARTKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library "); /* Use default options for METIS */ #if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5 options[0] = 0; METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &num_part, prt.part_sizes, options, &edgecut, prt.part); #else METIS_SetDefaultOptions(options); METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy, gr.vwgt, vsp.vsize, gr.ewgts, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part); #endif ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library"); } else { /* Sanity check: This should never happen! */ char msg[256]; sprintf(msg, "Unknown Metis algorithm %s.", alg); ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg); } } #endif /* ZOLTAN_METIS */ /* Get a time here */ if (get_times) times[2] = Zoltan_Time(zz->Timer); if (gr.final_output) { /* Do final output now because after the data will not be coherent: unscatter only unscatter part data, not graph */ ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr); } /* Ignore the timings of Final Ouput */ if (get_times) times[3] = Zoltan_Time(zz->Timer); ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr, geo, &prt, &vsp, NULL, &part); Zoltan_Third_Export_User(&part, num_imp, imp_gids, imp_lids, imp_procs, imp_to_part, num_exp, exp_gids, exp_lids, exp_procs, exp_to_part); /* Get a time here */ if (get_times) times[4] = Zoltan_Time(zz->Timer); if (get_times) Zoltan_Third_DisplayTime(zz, times); if (use_timers && timer_p >= 0) ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator); Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL); if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols); if (geo != NULL) ZOLTAN_FREE(&geo); ZOLTAN_FREE(&global_ids); ZOLTAN_FREE(&local_ids); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
/* * void Zoltan_Oct_gen_tree_from_input_data() * * This function will create a root node of on each processor which will * then be used to create an octree with regions associated with it. The * tree will then be balanced and the output used to balance "mesh regions" * on several processors. */ static void Zoltan_Oct_gen_tree_from_input_data(ZZ *zz, int oct_wgtflag, int *c1, int *c2, int *c3, float *c0, int createpartree) { char *yo = "Zoltan_Oct_gen_tree_from_input_data"; pRList RootList; /* list of all local roots */ pOctant RootOct; /* root octree octant */ COORD min, /* min coord bounds of objects */ max; /* max coord bounds of objects */ int num_extra; /* number of orphaned objects */ int num_objs; /* total number of local objects */ pRegion ptr, /* pointer to iterate trough region list */ ptr1; /* pointer to iterate trough region list */ pOctant root; /* root of the partition tree */ int i; /* index counter */ int count, /* count for leaf nodes in partition tree */ proc, /* proc leaf node of parition tree belongs to */ extra, /* extra leaf node flag, if not evenly divisible */ remainder; /* remainder of node, or processors to fill */ pOctant cursor, /* cursor to iterate through octant list */ cursor2, /* another cursor to iterate through octant list */ parent; /* parent of an octant */ int level, /* number of levels of refinement */ n, /* index counter */ part; /* partition counter */ Map *array; /* map of which processors own which octants */ int hold; /* used for calculating partition divisions */ int ierr = 0; #ifdef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN double bounds[6] = {DBL_MAX,DBL_MAX,DBL_MAX,-DBL_MAX,-DBL_MAX,-DBL_MAX}; COORD global_min, global_max; #endif /* KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN */ int nroots = 0; /*test*/ /* COORD gmin,gmax; */ OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure); ZOLTAN_TRACE_ENTER(zz, yo); /* * If there are no objects on this processor, do not create a root octant. * The partitioner will probably assign objects to this processor */ if(zz->Get_Num_Obj == NULL) { fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:", "Must register ZOLTAN_NUM_OBJ_FN function"); abort(); } *c3 = num_objs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr) { fprintf(stderr, "OCT [%d] %s: Error returned from user defined " "Get_Num_Obj function.\n", zz->Proc, yo); exit (-1); } ptr1 = NULL; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_get_bounds"); /* Need A Function To Get The Bounds Of The Local Objects */ Zoltan_Oct_get_bounds(zz, &ptr1, &num_objs, min, max, oct_wgtflag, c0); #ifndef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN /* For now, don't want to add the new query function to Zoltan. */ /* Zoltan_Oct_get_bounds appears to compute the global min and max from */ /* the object input. */ vector_set(OCT_info->OCT_gmin, min); vector_set(OCT_info->OCT_gmax, max); #else /*test*/ /*getMaxBounds(&gmin, &gmax);*/ if(zz->Get_Bounds_Geom == NULL) { fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:", "Must register Get_Bounds_Geom function"); abort(); } zz->Get_Bounds_Geom(zz->Get_Bounds_Geom_Data, bounds, &ierr); MPI_Allreduce(&(bounds[0]), &(global_min[0]), 3, MPI_DOUBLE, MPI_MIN, zz->Communicator); MPI_Allreduce(&(bounds[3]), &(global_max[0]), 3, MPI_DOUBLE, MPI_MAX, zz->Communicator); vector_set(OCT_info->OCT_gmin, global_min); vector_set(OCT_info->OCT_gmax, global_max); #endif /* * the following code segment was added to create a pseudo global octree * needed for the partitioner. The basic idea is to regroup all the * regions into something close to an octree partitioning and build the * tree from that. * NOTE: This way of doing things is very costly, especially when calling * this for the first time on a mesh not partitioned in an octree style * partitioning. */ level = 0; /* initialize level count */ /* * if more than 1 processor, need to find what level of refinement needed * to initially partition bounding box among the processors */ if(zz->Num_Proc > 1) { n = zz->Num_Proc; if(OCT_info->OCT_dimension == 2) hold = 4; else hold = 8; remainder = hold; for(; remainder > 0; level++) { int pr = (int)POW(hold, level); remainder = n - pr; } level--; } ZOLTAN_TRACE_DETAIL(zz, yo, "Before createpartree"); if(createpartree) { /* create the global root octant */ root = Zoltan_Oct_POct_new(OCT_info); Zoltan_Oct_setbounds(root, OCT_info->OCT_gmin, OCT_info->OCT_gmax); /* Zoltan_Oct_setOrientation(root, 0); */ /* subdivide to as many levels as calculated */ for(i=0; i<level; i++) { cursor = root; while(cursor != NULL) { if(Zoltan_Oct_isTerminal(cursor)) { cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor); Zoltan_Oct_terminal_refine(zz, cursor, 0); cursor = cursor2; } else cursor = Zoltan_Oct_POct_nextDfs(OCT_info, cursor); } } #if 0 if(zz->Proc == 0) for(i=0; i<8; i++) if(Zoltan_Oct_child(root, i) == NULL) fprintf(stderr,"NULL child pointer\n"); else fprintf(stderr, "child %d exists\n", i); #endif ZOLTAN_TRACE_DETAIL(zz, yo, "Before create map array"); /* this part creates the map array */ if(OCT_info->OCT_dimension == 2) { hold = (int)POW(4, level); /* ignoring the z+ octants */ if(hold == 0) hold = 1; } else hold = (int)POW(8, level); part = hold / zz->Num_Proc; /* how many octants per partition */ remainder = hold % zz->Num_Proc; /* extra octants, not evenly divisible */ extra = zz->Num_Proc - remainder;/* where to start adding extra octants */ array = (Map *) ZOLTAN_MALLOC(hold * sizeof(Map)); /* alloc map array */ if(array == NULL) { fprintf(stderr, "OCT ERROR on proc %d, could not allocate array map\n", zz->Proc); abort(); } /* initialize variables */ proc = 0; count = 0; i = 0; cursor = root; while(cursor != NULL) { cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor); if((Zoltan_Oct_isTerminal(cursor)) && (i < hold)) { if(proc == extra) { part++; extra = -1; } if(count != part) { array[i].npid = proc; array[i].list = RL_initRootList(); Zoltan_Oct_bounds(cursor, min, max); vector_set(array[i].min, min); vector_set(array[i].max, max); count++; } else { count = 1; proc++; array[i].npid = proc; array[i].list = RL_initRootList(); Zoltan_Oct_bounds(cursor, min, max); vector_set(array[i].min, min); vector_set(array[i].max, max); } if(proc == zz->Proc) { array[i].npid = -1; /* KDDKDD Added RL_freeList below. The * KDDKDD implementation from RPI leaked memory because the * KDDKDD test cases for setting array[i].list were not mutually * KDDKDD exclusive. Freeing the list produces the result we got * KDDKDD before, without the memory leak. */ /* LGG -- it seems to me that this array[i].list assignment is * not really necessary. It looks as though it has already been * assigned with the same information from the prev if-else * commented out RL_freeList(), and RL_initRootList() */ /*RL_freeList(&(array[i].list));*/ /* KDDKDD End addition */ /*array[i].list = RL_initRootList();*/ parent = Zoltan_Oct_parent(cursor); if(parent != NULL) Zoltan_Oct_setchild(parent, cursor->which, NULL); /* octant into local root list */ Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, -1); Zoltan_Oct_setMapIdx(cursor, i); nroots++; /* Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, zz->Proc); octant into local root list */ } i++; } cursor = cursor2; } RootList = Zoltan_Oct_POct_localroots(OCT_info); RootOct = RL_nextRootOctant(&RootList); if(RootOct != root) { /* KDDKDDFREE changed root to &root to allow root to be reset to NULL */ Zoltan_Oct_POct_delTree(OCT_info,&root); } OCT_info->map = array; OCT_info->mapsize = hold; } /* * attach the regions to the root... Zoltan_Oct_fix will create the octree * starting with the root and subdividing as needed */ num_extra = Zoltan_Oct_fix(zz, ptr1, num_objs); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_migreg_migrate_orphans"); Zoltan_Oct_migreg_migrate_orphans(zz, ptr1, num_extra, level, OCT_info->map, c1, c2); /* ZOLTAN_FREE(&array); */ while(ptr1 != NULL) { ptr = ptr1->next; ZOLTAN_FREE(&(ptr1->Global_ID)); ZOLTAN_FREE(&(ptr1->Local_ID)); ZOLTAN_FREE(&ptr1); ptr1 = ptr; } ZOLTAN_TRACE_EXIT(zz, yo); }
/* * void lb_oct_init(); * * initialize the calls needed to start the octree load balancing rounties */ static int lb_oct_init( ZZ *zz, /* The Zoltan structure with info for the OCTPART balancer. */ int *num_export, /* Number of non-local objs assigned to this processor in the new decomposition. */ ZOLTAN_ID_PTR *export_global_ids, /* Returned value: array of global IDs for non-local objects in this processor's new decomposition. */ ZOLTAN_ID_PTR *export_local_ids, /* Returned value: array of local IDs for non-local objects in this processor's new decomposition. */ int **export_procs, /* Returned value: array of processor IDs for processors owning the non-local objects in this processor's new decomposition. */ int **export_to_part, /* Returned value: array of partitions to which objects are imported. KDDKDD Assume #parts==#procs. */ int oct_dim, /* Dimension of method (2D or 3D) */ int oct_method, /* Flag specifying curve to be used. */ int oct_maxoctregions, /* max # of objects in leaves of octree. */ int oct_minoctregions, /* min # of objects in leaves of octree. */ int oct_output_level, /* Flag specifying amount of output. */ int oct_wgtflag, /* Flag specifying use of object weights. */ float *part_sizes /* Array of size zz->Num_Global_Parts containing the percentage of work to be assigned to each partition. */ ) { char *yo = "lb_oct_init"; OCT_Global_Info *OCT_info; int nsentags; /* number of tags being sent */ pRegion export_regs; /* */ int nrectags; /* number of tags received */ int kk; double time1,time2; /* timers */ double timestart,timestop; /* timers */ double timers[4]; /* diagnostic timers 0 = start-up time before recursion 1 = time before median iterations 2 = time in median iterations 3 = communication time */ int counters[6]; /* diagnostic counts 0 = # of median iterations 1 = # of objects sent 2 = # of objects received 3 = most objects this proc ever owns */ float c[4]; int createpartree = 0; /*int num_gid_entries = zz->Num_GID;*/ /*int num_lid_entries = zz->Num_LID;*/ ZOLTAN_TRACE_ENTER(zz, yo); MPI_Barrier(zz->Communicator); timestart = MPI_Wtime(); /* initialize timers and counters */ counters[0] = 0; counters[1] = 0; counters[2] = 0; counters[3] = 0; counters[4] = 0; counters[5] = 0; c[0] = 0; c[1] = 0; c[2] = 0; c[3] = 0; timers[1] = 0.0; timers[2] = 0.0; timers[3] = 0.0; nsentags = nrectags = 0; if(zz->LB.Data_Structure == NULL) { OCT_info = Zoltan_Oct_POct_init(zz, zz->Proc, oct_dim); Zoltan_Oct_set_method(OCT_info, oct_method); Zoltan_Oct_set_maxregions(oct_maxoctregions); Zoltan_Oct_set_minregions(oct_minoctregions); createpartree = 1; } else { OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure); } /* create the octree structure */ time1 = MPI_Wtime(); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_gen_tree_from_input_data"); Zoltan_Oct_gen_tree_from_input_data(zz, oct_wgtflag, &counters[1], &counters[2], &counters[3], &c[0], createpartree); time2 = MPI_Wtime(); timers[0] = time2 - time1; /* time took to create octree */ /* Zoltan_Oct_POct_printResults(OCT_info); */ /* partition the octree structure */ time1 = MPI_Wtime(); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_partition"); /* old call to dfs_paritition: */ #if 0 Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1]); #else /*************************** if(zz->Proc == 0) { int debug_i; for(debug_i=0; debug_i<zz->Num_Proc; debug_i++) { fprintf(stdout,"Part_size[%d] = %f\n", debug_i, part_sizes[debug_i]); } } ****************************/ Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1], part_sizes); #endif time2 = MPI_Wtime(); timers[1] = time2 - time1; /* time took to partition octree */ if (oct_output_level > 2) { Zoltan_Oct_Plots(zz); } /* set up tags for migrations */ time1 = MPI_Wtime(); #if 0 /* KDDKDD -- Count is never used; why is it computed? */ { pRList RootList; /* list of all local roots */ pOctant RootOct; /* root octree octant */ int count = 0; RootList = Zoltan_Oct_POct_localroots(OCT_info); while((RootOct = RL_nextRootOctant(&RootList))) { while(RootOct) { if(Zoltan_Oct_isTerminal(RootOct)) { count += Zoltan_Oct_nRegions(RootOct); } RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct); } } } #endif ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_migrate"); Zoltan_Oct_dfs_migrate(zz, &nsentags, &export_regs, &nrectags, &c[2], &c[3], &counters[3], &counters[5]); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_fix_tags"); if (zz->LB.Return_Lists) { *num_export = nrectags; if (nrectags > 0) Zoltan_Oct_fix_tags(zz, export_global_ids, export_local_ids, export_procs, export_to_part, nrectags, export_regs); } time2 = MPI_Wtime(); timers[2] = time2 - time1; /* time took to setup migration */ #if 0 /* KDDKDD -- Count is never used; why is it computed? */ { /* count the number of objects on this processor */ pRList RootList; /* list of all local roots */ pOctant RootOct; /* root octree octant */ int count = 0; RootList = Zoltan_Oct_POct_localroots(OCT_info); while((RootOct = RL_nextRootOctant(&RootList))) { while(RootOct) { if(Zoltan_Oct_isTerminal(RootOct)) { count += Zoltan_Oct_nRegions(RootOct); } RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct); } } } #endif counters[4] = nsentags; MPI_Barrier(zz->Communicator); timestop = MPI_Wtime(); if (oct_output_level > 0) { ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_print_stats"); Zoltan_Oct_print_stats(zz, timestop-timestart, timers, counters, c, oct_output_level); } for (kk = 0; kk < nrectags; kk++) { ZOLTAN_FREE(&(export_regs[kk].Global_ID)); ZOLTAN_FREE(&(export_regs[kk].Local_ID)); } ZOLTAN_FREE(&export_regs); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_global_clear"); Zoltan_Oct_global_clear(OCT_info); /* KDDKDD Don't understand how re-used octree will work, especially without * KDDKDD the Zoltan_Oct_Bounds_Geom function. For now, we'll delete everything; * KDDKDD we can move back to saving some of the tree later. */ Zoltan_Oct_Free_Structure(zz); /* KDDKDD END */ /* Temporary return value until error codes are fully implemented. */ ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_OK); }
int Zoltan_Order( ZZ *zz, /* Zoltan structure */ int num_gid_entries, /* # of entries for a global id */ int num_obj, /* Number of objects to order */ ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */ /* The application must allocate enough space */ int *rank, /* rank[i] is the rank of gids[i] */ int *iperm /* iperm[rank[i]]=i, only for sequential ordering */ ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids or enough space to store such a list * lids, a list of local ids or enough space to store such a list * Output: * num_gid_entries * num_lid_entries * gids, a list of global ids (filled in if empty on entry) * lids, a list of local ids (filled in if empty on entry) * rank, rank[i] is the global rank of gids[i] * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; int * vtxdist = NULL; ZOLTAN_ID_PTR local_gids=NULL, lids=NULL; int local_num_obj; int *local_rank = NULL, *local_iperm=NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = gcomm[0]; if (num_gid_entries != zz->Num_GID) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "num_gid_entries doesn't have the good value"); return (ZOLTAN_FATAL); } zz->Order.nbr_objects = num_obj; zz->Order.rank = rank; zz->Order.iperm = iperm; zz->Order.gids = gids; zz->Order.lids = lids; zz->Order.start = NULL; zz->Order.ancestor = NULL; zz->Order.leaves = NULL; zz->Order.nbr_leaves = 0; zz->Order.nbr_blocks = 0; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); #ifdef HAVE_MPI strncpy(opt.order_type, "DIST", MAX_PARAM_STRING_LEN); #else strncpy(opt.order_type, "SERIAL", MAX_PARAM_STRING_LEN); #endif /* HAVE_MPI */ opt.use_order_info = 0; opt.start_index = 0; opt.reorder = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type); Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index); Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); zz->Order.start_index = opt.start_index; /* * Check that the user has allocated space for the return args. */ if (!(gids && rank)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } #ifdef ZOLTAN_PARMETIS else if (!strcmp(opt.method, "NODEND")) { Order_fn = Zoltan_ParMetis_Order; } else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "GLOBAL"); } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_SCOTCH else if (!strcmp(opt.method, "SCOTCH")) { Order_fn = Zoltan_Scotch_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); /* strcpy(opt.order_type, "GLOBAL"); */ } #endif /* ZOLTAN_SCOTCH */ else { fprintf(stderr, "%s\n", opt.method); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } if (!strcmp(opt.order_type, "GLOBAL")) strcpy (opt.order_type, "DIST"); if (!strcmp(opt.order_type, "LOCAL")) strcpy (opt.order_type, "SERIAL"); strcpy(zz->Order.order_type, opt.order_type); /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /* * Call the actual ordering function. * Compute gid according to the local graph. */ if (zz->Get_Num_Obj != NULL) { local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); return (ierr); } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); return (ZOLTAN_FATAL); } local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj); local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); local_iperm = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj); ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, local_iperm, &opt); ZOLTAN_FREE(&lids); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN) { ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); Zoltan_Multifree(__FILE__, __LINE__, 3, &local_gids, &local_rank, &local_iperm); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* Compute inverse permutation if necessary */ if ((!(opt.return_args & RETURN_RANK) && (rank != NULL)) || (!(opt.return_args & RETURN_IPERM) && (iperm != NULL))) { ierr = Zoltan_Get_Distribution(zz, &vtxdist); if (ierr) { /* Error */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n"); return (ierr); } if (!(opt.return_args & RETURN_RANK) && (rank != NULL)) { /* Compute rank from iperm */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, local_iperm, local_rank, vtxdist, opt.order_type, opt.start_index); } else if (!(opt.return_args & RETURN_IPERM) && (iperm != NULL)) { /* Compute iperm from rank */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, local_rank, local_iperm, vtxdist, opt.order_type, opt.start_index); } ZOLTAN_FREE(&vtxdist); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done Invert Permutation"); /* TODO: Use directly the "graph" structure to avoid to duplicate things. */ /* I store : GNO, rank, iperm */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, (local_iperm==NULL)?0:1, local_num_obj, 0); /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj); /* Associate all the data with our xGNO */ Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, (ZOLTAN_ID_PTR) local_iperm, NULL, local_num_obj); ZOLTAN_FREE(&local_gids); ZOLTAN_FREE(&local_rank); ZOLTAN_FREE(&local_iperm); Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)rank, (ZOLTAN_ID_PTR)iperm, NULL, num_obj, NULL); Zoltan_DD_Destroy(&dd); ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < num_obj; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)])); printf(", rank = %3d\n", rank[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } ZOLTAN_TRACE_EXIT(zz, yo); if (ierr) return (ierr); else return (ZOLTAN_OK); }
int Zoltan_Order ( struct Zoltan_Struct *zz, int num_gid_entries, int num_obj, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR permuted_global_ids ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids. * num_gid_entries * Output: * permuted_global_ids * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; ZOLTAN_ID_PTR local_gids=NULL, lids=NULL; int local_num_obj; int *local_rank = NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = gcomm[0]; if (num_gid_entries != zz->Num_GID) { char msg[253]; sprintf(msg, "num_gid_entries=%d is not equal to parameter setting " "NUM_GID_ENTRIES=%d\n", num_gid_entries, zz->Num_GID); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); return (ZOLTAN_FATAL); } zz->Order.nbr_objects = num_obj; zz->Order.start = NULL; zz->Order.ancestor = NULL; zz->Order.leaves = NULL; zz->Order.nbr_leaves = 0; zz->Order.nbr_blocks = 0; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ #ifdef HAVE_MPI strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); strcpy(zz->Order.order_type, "GLOBAL"); #else strncpy(opt.method, "METIS", MAX_PARAM_STRING_LEN); strcpy(zz->Order.order_type, "LOCAL"); #endif /* HAVE_MPI */ opt.use_order_info = 0; opt.start_index = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); /* * Check that the user has allocated space for the return args. */ if (num_obj && !(gids && permuted_global_ids)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } else if (!strcmp(opt.method, "LOCAL_HSFC")) { Order_fn = Zoltan_LocalHSFC_Order; strcpy(zz->Order.order_type, "LOCAL"); /*MMW, not sure about this*/ } #ifdef ZOLTAN_PARMETIS else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; strcpy(zz->Order.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; strcpy(zz->Order.order_type, "GLOBAL"); } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_SCOTCH else if (!strcmp(opt.method, "SCOTCH")) { Order_fn = Zoltan_Scotch_Order; strcpy(zz->Order.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PTSCOTCH")) { Order_fn = Zoltan_Scotch_Order; strcpy(zz->Order.order_type, "GLOBAL"); } #endif /* ZOLTAN_SCOTCH */ #ifdef ZOLTAN_HUND else if (!strcasecmp(opt.method, "HUND")) { ierr = Zoltan_HUND(zz, num_gid_entries, num_obj, gids, permuted_global_ids, NULL); goto End; } #endif /* ZOLTAN_HUND */ else { fprintf(stderr, "%s\n", opt.method); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* TODO : Ask why useful ! */ /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /************************************ * Check for required query function ************************************/ if (zz->Get_Num_Obj != NULL) { local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); return (ierr); } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); return (ZOLTAN_FATAL); } /* TODO allocate all this stuff with the graph */ local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj); local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj); /* * Call the actual ordering function. * Compute gid according to the local graph. */ ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, NULL, &opt); ZOLTAN_FREE(&lids); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN) { ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); Zoltan_Multifree(__FILE__, __LINE__, 2, &local_gids, &local_rank); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* TODO: Use directly the "graph" structure to avoid to duplicate things. */ /* TODO: At this time, I consider rank == permuted_global_ids */ /* I store : GNO, rank, permuted GID */ /* MMW: perhaps don't ever use graph here since we need to support geometric orderings, otherwise need if/else */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, 0, local_num_obj, 0); /* Hope a linear assignment will help a little */ if (local_num_obj) Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj); /* Associate all the data with our xGNO */ Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, NULL, NULL, local_num_obj); ZOLTAN_FREE(&local_gids); ZOLTAN_FREE(&local_rank); Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)permuted_global_ids, NULL, NULL, num_obj, NULL); Zoltan_DD_Destroy(&dd); ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < num_obj; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)])); printf(", rank = %3d\n", permuted_global_ids[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } #ifdef ZOLTAN_HUND End: #endif /*ZOLTAN_HUND*/ ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }