void octree::compute_properties_double(tree_structure &tree) { /***************************************************** Assign the memory buffers, note that we check the size first and if needed we increase the size of the generalBuffer1 Size required: - multipoleD -> double4*3_n_nodes -> 6*n_nodes*uint4 - lower/upperbounds -> 2*n_nodes*uint4 - node lower/upper -> 2*n_nodes*uint4 - SUM: 10*n_nodes*uint4 - generalBuffer1 has default size: 3*N*uint4 check if 10*n_nodes < 3*N if so realloc *****************************************************/ if(10*tree.n_nodes > 3*tree.n) { fprintf(stderr, "Resizeing the generalBuffer1 \n"); tree.generalBuffer1.cresize(8*tree.n_nodes*4, false); } my_dev::dev_mem<double4> multipoleD(devContext); multipoleD.cmalloc_copy(tree.generalBuffer1.get_pinned(), tree.generalBuffer1.get_flags(), tree.generalBuffer1.get_devMem(), &tree.generalBuffer1[0], 0, 3*tree.n_nodes, getAllignmentOffset(0)); //Offset is in uint, so: double4 = 8uint*3*n_nodes tree.nodeLowerBounds.cmalloc_copy(tree.generalBuffer1.get_pinned(), tree.generalBuffer1.get_flags(), tree.generalBuffer1.get_devMem(), &tree.generalBuffer1[8*3*tree.n_nodes], 8*3*tree.n_nodes, tree.n_nodes, getAllignmentOffset(8*3*tree.n_nodes)); int prevOffsetSum = getAllignmentOffset(8*3*tree.n_nodes); //The offset of output tree.nodeUpperBounds.cmalloc_copy(tree.generalBuffer1.get_pinned(), tree.generalBuffer1.get_flags(), tree.generalBuffer1.get_devMem(), &tree.generalBuffer1[8*3*tree.n_nodes + 4*tree.n_nodes], 8*3*tree.n_nodes + 4*tree.n_nodes, tree.n_nodes, prevOffsetSum + getAllignmentOffset(8*3*tree.n_nodes + 4*tree.n_nodes + prevOffsetSum)); //Computes the tree-properties (size, cm, monopole, quadropole, etc) //start the kernel for the leaf-type nodes propsLeafD.set_arg<int>(0, &tree.n_leafs); propsLeafD.set_arg<cl_mem>(1, tree.leafNodeIdx.p()); propsLeafD.set_arg<cl_mem>(2, tree.node_bodies.p()); propsLeafD.set_arg<cl_mem>(3, tree.bodies_Ppos.p()); propsLeafD.set_arg<cl_mem>(4, multipoleD.p()); propsLeafD.set_arg<cl_mem>(5, tree.nodeLowerBounds.p()); propsLeafD.set_arg<cl_mem>(6, tree.nodeUpperBounds.p()); propsLeafD.set_arg<cl_mem>(7, tree.bodies_Pvel.p()); //Velocity to get max eps propsLeafD.setWork(tree.n_leafs, 128); printf("PropsLeaf: "); propsLeafD.printWorkSize(); propsLeafD.execute(); int temp = tree.n_nodes-tree.n_leafs; propsNonLeafD.set_arg<int>(0, &temp); propsNonLeafD.set_arg<cl_mem>(1, tree.leafNodeIdx.p()); propsNonLeafD.set_arg<cl_mem>(2, tree.node_level_list.p()); propsNonLeafD.set_arg<cl_mem>(3, tree.n_children.p()); propsNonLeafD.set_arg<cl_mem>(4, multipoleD.p()); propsNonLeafD.set_arg<cl_mem>(5, tree.nodeLowerBounds.p()); propsNonLeafD.set_arg<cl_mem>(6, tree.nodeUpperBounds.p()); //Work from the bottom up for(int i=tree.n_levels; i >= 1; i--) { propsNonLeafD.set_arg<int>(0, &i); { vector<size_t> localWork(2), globalWork(2); int totalOnThisLevel; totalOnThisLevel = tree.node_level_list[i]-tree.node_level_list[i-1]; propsNonLeafD.setWork(totalOnThisLevel, 128); printf("PropsNonLeaf, nodes on level %d : %d (start: %d end: %d) , config: \t", i, totalOnThisLevel, tree.node_level_list[i-1], tree.node_level_list[i]); propsNonLeafD.printWorkSize(); } propsNonLeafD.set_arg<int>(0, &i); //set the level propsNonLeafD.execute(); } float theta2 = theta; propsScalingD.set_arg<int>(0, &tree.n_nodes); propsScalingD.set_arg<real4>(1, &tree.corner); propsScalingD.set_arg<cl_mem>(2, multipoleD.p()); propsScalingD.set_arg<cl_mem>(3, tree.nodeLowerBounds.p()); propsScalingD.set_arg<cl_mem>(4, tree.nodeUpperBounds.p()); propsScalingD.set_arg<cl_mem>(5, tree.n_children.p()); propsScalingD.set_arg<cl_mem>(6, tree.multipole.p()); propsScalingD.set_arg<float >(7, &theta2); propsScalingD.set_arg<cl_mem>(8, tree.boxSizeInfo.p()); propsScalingD.set_arg<cl_mem>(9, tree.boxCenterInfo.p()); propsScalingD.set_arg<cl_mem>(10, tree.node_bodies.p()); propsScalingD.setWork(tree.n_nodes, 128); printf("propsScaling: \t "); propsScalingD.printWorkSize(); propsScalingD.execute(); #ifdef INDSOFT //If we use individual softening we need to get the max softening value //to be broadcasted during the exchange of the LET boundaries. //Only copy the root node that contains the max value my_dev::dev_stream memCpyStream; tree.multipole.d2h(3, false, memCpyStream.s()); #endif //Set the group properties, note that it is not based on the nodes anymore //but on self created groups based on particle order setPHGroupData copyNodeDataToGroupData.set_arg<int>(0, &tree.n_groups); copyNodeDataToGroupData.set_arg<int>(1, &tree.n); copyNodeDataToGroupData.set_arg<cl_mem>(2, tree.bodies_Ppos.p()); copyNodeDataToGroupData.set_arg<cl_mem>(3, tree.group_list_test.p()); copyNodeDataToGroupData.set_arg<cl_mem>(4, tree.groupCenterInfo.p()); copyNodeDataToGroupData.set_arg<cl_mem>(5, tree.groupSizeInfo.p()); copyNodeDataToGroupData.setWork(-1, NCRIT, tree.n_groups); copyNodeDataToGroupData.printWorkSize(); copyNodeDataToGroupData.execute(); #ifdef INDSOFT memCpyStream.sync(); this->maxLocalEps = tree.multipole[0*3 + 1].w; //Softening value #else #endif //Get the local domain boundary based on group positions and sizes real4 r_min, r_max; getBoundariesGroups(tree, r_min, r_max); #if 0 //Write the tree structure to file string nodeFileName = "fullTreeStructure.txt"; char fileName[256]; sprintf(fileName, "fullTreeStructure-%d.txt", mpiGetRank()); ofstream nodeFile; //nodeFile.open(nodeFileName.c_str()); nodeFile.open(fileName); tree.multipole.d2h(); tree.boxSizeInfo.d2h(); tree.boxCenterInfo.d2h(); for(int i=0; i < tree.n_nodes; i++) { //nodeFile << i << "\t" << tree.boxCenterInfo[i].x << "\t" << tree.boxCenterInfo[i].y; //nodeFile << "\t" << 2*tree.boxSizeInfo[i].x << "\t" << 2*tree.boxSizeInfo[i].y << "\t"; nodeFile << i << "\t" << tree.boxCenterInfo[i].x-tree.boxSizeInfo[i].x << "\t" << tree.boxCenterInfo[i].y-tree.boxSizeInfo[i].y; nodeFile << "\t" << tree.boxCenterInfo[i].x+tree.boxSizeInfo[i].x << "\t" << tree.boxCenterInfo[i].y+tree.boxSizeInfo[i].y << "\t"; nodeFile << tree.multipole[i*3+0].x << "\t" << tree.multipole[i*3+0].w << "\n"; } nodeFile.close(); sprintf(fileName, "fullTreeStructureParticles-%d.txt", mpiGetRank()); ofstream partFile; partFile.open(fileName); tree.bodies_pos.d2h(); for(int i=0; i < tree.n; i++) { float4 pos = tree.bodies_pos[i]; partFile << i << "\t" << pos.x << "\t" << pos.y << "\t" << pos.z << endl; } partFile.close(); #endif }
void octree::approximate_dust(tree_structure &tree) { if(tree.n_dust == 0) return; uint2 node_begend; int level_start = 2; node_begend.x = tree.level_list[level_start].x; node_begend.y = tree.level_list[level_start].y; //Reset the active particles tree.active_dust_list.zeroMem(); //Set the kernel parameters, many! approxGrav.set_arg<int>(0, &tree.n_dust_groups); approxGrav.set_arg<int>(1, &tree.n_dust); approxGrav.set_arg<float>(2, &(this->eps2)); approxGrav.set_arg<uint2>(3, &node_begend); approxGrav.set_arg<cl_mem>(4, tree.activeDustGrouplist.p()); approxGrav.set_arg<cl_mem>(5, tree.bodies_Ppos.p()); //Bodies from the tree approxGrav.set_arg<cl_mem>(6, tree.multipole.p()); approxGrav.set_arg<cl_mem>(7, tree.dust_acc1.p()); approxGrav.set_arg<cl_mem>(8, tree.dust_pos.p()); //Dust bodies approxGrav.set_arg<cl_mem>(9, tree.dust_ngb.p()); approxGrav.set_arg<cl_mem>(10, tree.active_dust_list.p()); approxGrav.set_arg<cl_mem>(11, tree.dust_interactions.p()); approxGrav.set_arg<cl_mem>(12, tree.boxSizeInfo.p()); approxGrav.set_arg<cl_mem>(13, tree.dust_groupSizeInfo.p()); approxGrav.set_arg<cl_mem>(14, tree.boxCenterInfo.p()); approxGrav.set_arg<cl_mem>(15, tree.dust_groupCenterInfo.p()); approxGrav.set_arg<cl_mem>(16, tree.dust_vel.p()); approxGrav.set_arg<cl_mem>(17, tree.generalBuffer1.p()); //Instead of using Local memory approxGrav.set_arg<real4>(18, tree.boxSizeInfo, 4, "texNodeSize"); approxGrav.set_arg<real4>(19, tree.boxCenterInfo, 4, "texNodeCenter"); approxGrav.set_arg<real4>(20, tree.multipole, 4, "texMultipole"); approxGrav.set_arg<real4>(21, tree.dust_pos, 4, "texBody"); approxGrav.setWork(-1, NTHREAD, nBlocksForTreeWalk); approxGrav.execute(gravStream->s()); //First half //Print interaction statistics #if 0 tree.dust_interactions.d2h(); long long directSum = 0; long long apprSum = 0; long long directSum2 = 0; long long apprSum2 = 0; int maxDir = -1; int maxAppr = -1; for(int i=0; i < tree.n_dust; i++) { apprSum += tree.dust_interactions[i].x; directSum += tree.dust_interactions[i].y; maxAppr = max(maxAppr,tree.dust_interactions[i].x); maxDir = max(maxDir,tree.dust_interactions[i].y); apprSum2 += tree.dust_interactions[i].x*tree.dust_interactions[i].x; directSum2 += tree.dust_interactions[i].y*tree.dust_interactions[i].y; } cout << "DUST Interaction at (rank= " << mpiGetRank() << " ) iter: " << iter << "\tdirect: " << directSum << "\tappr: " << apprSum << "\t"; cout << "avg dir: " << directSum / tree.n_dust << "\tavg appr: " << apprSum / tree.n_dust << "\tMaxdir: " << maxDir << "\tmaxAppr: " << maxAppr << endl; cout << "sigma dir: " << sqrt((directSum2 - directSum)/ tree.n_dust) << "\tsigma appr: " << std::sqrt((apprSum2 - apprSum) / tree.n_dust) << endl; #endif /* tree.dust_acc1.d2h(); tree.dust_pos.d2h(); for(int i=0; i < tree.n_dust; i++) { fprintf(stderr,"%d\t%f %f %f %f \t %f %f %f\n", i, tree.dust_acc1[i].x, tree.dust_acc1[i].y, tree.dust_acc1[i].z, tree.dust_acc1[i].w, tree.dust_pos[i].x, tree.dust_pos[i].y, tree.dust_pos[i].z); } */ //exit(0); }
void octree::build (tree_structure &tree) { int level = 0; int validCount = 0; int offset = 0; /******** load kernels **********/ /******** create memory buffers **********/ my_dev::dev_mem<uint> validList(devContext); my_dev::dev_mem<uint> compactList(devContext); validList.cmalloc_copy(tree.generalBuffer1.get_pinned(), tree.generalBuffer1.get_flags(), tree.generalBuffer1.get_devMem(), &tree.generalBuffer1[0], 0, tree.n*2, getAllignmentOffset(0)); validList.zeroMem(); compactList.cmalloc_copy(tree.generalBuffer1.get_pinned(), tree.generalBuffer1.get_flags(), tree.generalBuffer1.get_devMem(), &tree.generalBuffer1[tree.n*2], tree.n*2, tree.n*2, getAllignmentOffset(tree.n*2)); /******** set kernels parameters **********/ build_key_list.set_arg<cl_mem>(0, tree.bodies_key.p()); build_key_list.set_arg<cl_mem>(1, tree.bodies_Ppos.p()); build_key_list.set_arg<int>(2, &tree.n); build_key_list.set_arg<real4>(3, &tree.corner); build_key_list.setWork(tree.n, 128); build_valid_list.set_arg<int>(0, &tree.n); build_valid_list.set_arg<int>(1, &level); build_valid_list.set_arg<cl_mem>(2, tree.bodies_key.p()); build_valid_list.set_arg<cl_mem>(3, validList.p()); build_valid_list.setWork(tree.n, 128); build_nodes.set_arg<int>(0, &level); build_nodes.set_arg<int>(1, &validCount); build_nodes.set_arg<int>(2, &offset); build_nodes.set_arg<cl_mem>(3, compactList.p()); build_nodes.set_arg<cl_mem>(4, tree.bodies_key.p()); build_nodes.set_arg<cl_mem>(5, tree.node_key.p()); build_nodes.set_arg<cl_mem>(6, tree.n_children.p()); build_nodes.set_arg<cl_mem>(7, tree.node_bodies.p()); link_tree.set_arg<int>(0, &offset); link_tree.set_arg<cl_mem>(1, tree.n_children.p()); link_tree.set_arg<cl_mem>(2, tree.node_bodies.p()); link_tree.set_arg<cl_mem>(3, tree.bodies_Ppos.p()); link_tree.set_arg<real4>(4, &tree.corner); link_tree.set_arg<cl_mem>(5, tree.level_list.p()); link_tree.set_arg<cl_mem>(6, validList.p()); link_tree.set_arg<cl_mem>(7, tree.node_key.p()); link_tree.set_arg<cl_mem>(8, tree.bodies_key.p()); link_tree.set_arg<int>(9, &level); /********** build list of keys ********/ build_key_list.execute(); /****** build the levels *********/ int nodeSum = 0; for (level = 0; level < MAXLEVELS; level++) { // mark bodies to be combined into nodes build_valid_list.set_arg<int>(1, &level); build_valid_list.execute(); //gpuCompact to get number of created nodes gpuCompact(devContext, validList, compactList, tree.n*2, &validCount); nodeSum += validCount / 2; printf("ValidCount (%d): %d \tSum: %d Offset: %d\n", mpiGetRank(), validCount, nodeSum, offset); validCount /= 2; if (validCount == 0) break; // asssemble nodes build_nodes.setWork(validCount, 128); build_nodes.set_arg<int>(0, &level); build_nodes.set_arg<int>(1, &validCount); build_nodes.set_arg<int>(2, &offset); build_nodes.execute(); tree.level_list[level] = (uint2){offset, offset + validCount}; offset += validCount; } //end for lvl //Put the last level + 1 index to 0,0 //so we dont need an extra if statement in the linking phase tree.level_list[level] = (uint2){0, 0}; tree.level_list.h2d(); int n_nodes = offset; tree.n_nodes = n_nodes; /***** Link the tree ******/ link_tree.set_arg<int>(0, &offset); //Offset=number of nodes link_tree.set_arg<int>(9, &level); //level=highest number of levels //The maximum number of levels that can be used is MAXLEVEl //if max level is larger than that the program will exit printf("Max level : %d \n", level); if(level >= MAXLEVELS) { cerr << "The tree has become too deep, the program will exit. \n"; cerr << "Consider the removal of far away particles to prevent a too large box. \n"; maxlevels_exceeded = true; return; //exit(0); } link_tree.setWork(n_nodes, 128); printf("Link_tree: "); link_tree.printWorkSize(); tree.n_levels = level-1; for(int i=0; i < level; i++) printf("%d\t%d\t%d\n", i, tree.level_list[i].x, tree.level_list[i].y); //Link the tree link_tree.execute(); //After executing link_tree, the id_list contains for each node //the ID of its parent. //Valid_list contains for each node if its a leaf (valid) or a normal //node -> non_valid //Execute a split on the validList to get seperate id lists //for the leafs and nodes. Used when computing multipoles tree.leafNodeIdx.cmalloc(tree.n_nodes , false); //Split the leaf ids and non-leaf node ids gpuSplit(devContext, validList, tree.leafNodeIdx, tree.n_nodes, &tree.n_leafs); printf("Total nodes: %d N_leafs: %d non-leafs: %d \n", tree.n_nodes, tree.n_leafs, tree.n_nodes - tree.n_leafs); build_level_list.set_arg<int>(0, &tree.n_nodes); build_level_list.set_arg<int>(1, &tree.n_leafs); build_level_list.set_arg<cl_mem>(2, tree.leafNodeIdx.p()); build_level_list.set_arg<cl_mem>(3, tree.node_bodies.p()); build_level_list.set_arg<cl_mem>(4, validList.p()); build_level_list.setWork(tree.n_nodes-tree.n_leafs, 128); validList.zeroMem(); //Build the level list based on the leafIdx list //required for easy access in the compute node properties build_level_list.execute(); tree.node_level_list.cmalloc(level*2 , false); int levelThing; gpuCompact(devContext, validList, tree.node_level_list, 2*(tree.n_nodes-tree.n_leafs), &levelThing); tree.node_level_list.d2h(); //We only care about end positions, so compress the list: int j=0; for(int i=0; i < levelThing; i+=2, j++) tree.node_level_list[j] = tree.node_level_list[i]; tree.node_level_list[j] =tree.node_level_list[levelThing-1]+1; //Add 1 to make it the end position levelThing = j+1; tree.node_level_list.h2d(); printf("Finished level list \n"); for(int i=0; i < levelThing; i++) { printf("node_level_list: %d \t%d\n", i, tree.node_level_list[i]); } ///****** Start building the particle groups *******/////// //Compute the box size, the max length of one of the sides of the rectangle real size = fmax(fabs(rMaxLocalTree.z - rMinLocalTree.z), fmax(fabs(rMaxLocalTree.y - rMinLocalTree.y), fabs(rMaxLocalTree.x - rMinLocalTree.x))); real dist = ((rMaxLocalTree.z - rMinLocalTree.z) * (rMaxLocalTree.z - rMinLocalTree.z) + (rMaxLocalTree.y - rMinLocalTree.y) * (rMaxLocalTree.y - rMinLocalTree.y) + (rMaxLocalTree.x - rMinLocalTree.x) * (rMaxLocalTree.x - rMinLocalTree.x)); float maxDist = sqrt(dist) / 10; maxDist *= maxDist; //Square since we dont do sqrt on device fprintf(stderr,"Box max size: %f en max dist: %f \t %f en %f \n", size, dist, sqrt(dist), maxDist); //maxDist = 50; validList.zeroMem(); //The newest group creation method! define_groups.set_arg<int>(0, &tree.n); define_groups.set_arg<cl_mem>(1, validList.p()); define_groups.set_arg<cl_mem>(2, tree.bodies_Ppos.p()); define_groups.set_arg<float>(3, &maxDist); define_groups.setWork(tree.n, 128); define_groups.execute(); //gpuCompact gpuCompact(devContext, validList, compactList, tree.n*2, &validCount); printf("Found number of groups: %d \n", validCount/2); tree.n_groups = validCount/2; //Now compact validList to get the list of group ids tree.group_list_test.cmalloc(tree.n_groups , false); store_groups.set_arg<int>(0, &tree.n); store_groups.set_arg<int>(1, &tree.n_groups); store_groups.set_arg<cl_mem>(2, compactList.p()); store_groups.set_arg<cl_mem>(3, tree.body2group_list.p()); store_groups.set_arg<cl_mem>(4, tree.group_list_test.p()); store_groups.setWork(-1, NCRIT, tree.n_groups); store_groups.execute(); //Memory allocation for the valid group lists if(tree.active_group_list.get_size() > 0) { tree.active_group_list.cresize(tree.n_groups, false); tree.activeGrpList.cresize(tree.n_groups, false); } else { tree.active_group_list.cmalloc(tree.n_groups, false); tree.activeGrpList.cmalloc(tree.n_groups, false); } printf("Tree built complete!\n"); /*************************/ }