//BOPI // !IROUTINE: setSizes - Establish the message sizes // // !INTERFACE: void SparseMsgVM::setSizes( // // !RETURN VALUE: // // !ARGUMENTS: UInt *sizes) // // !DESCRIPTION: // Establish and share the size of a message between pet(i,j). Also // sets aside memory for this messages in the form of send/receive // buffers. // //EOPI // !REQUIREMENTS: //----------------------------------------------------------------------------- { // Error checking if (obj_state != PATTERN) throw Ex() << "SparseMsgVM illegal transition from state:" << obj_state << " to SIZE"; // First, set up send buffers UInt totalsize = 0; // Find total buffer size; round up to word boundaries // for each new buffer for (UInt i = 0; i < nsend; i++) { totalsize += round_to_dword(sizes[i]); } sendBuf = new UChar[totalsize+1]; // Set up pointers into buffer UInt cur_loc = 0; for (UInt i = 0; i < nsend; i++) { UInt bsize = round_to_dword(sizes[i]); buffer &buf = outBuffers[i]; buf.beg = outBuffers[i].cur = &sendBuf[cur_loc]; buf.end = &sendBuf[cur_loc+bsize]; buf.bsize = bsize; buf.msize = sizes[i]; cur_loc += bsize; petToOutBuffer[buf.pet] = &buf; } //std::cout << "last buf end:" << (int) outBuffers[nsend-1].end << ", sendbuf end:" << (int) &sendBuf[cur_loc] << std::endl; // Second, send sizes to receive // avoid allocating zero (add 1) //std::vector<MPI_Request> request(num_incoming+1, NULL); //std::vector<MPI_Status> status(num_incoming+1); std::vector<ESMCI::VMK::commhandle*> commhp(num_incoming+1); for (UInt i = 0; i < num_incoming; i++) commhp[i] = new ESMCI::VMK::commhandle; // Post Recieves std::vector<int> inSizes(num_incoming+1); UInt tag0 = 0; UInt enD = num_incoming - (sendself ? 1 : 0); for (UInt i = 0; i < enD; i++) { //MPI_Irecv(&inSizes[i], 1, MPI_INT, MPI_ANY_SOURCE, tag0, comm, &request[i]); vm.recv(static_cast<void*>(&inSizes[i]), sizeof(int), VM_ANY_SRC, &commhp[i], tag0); } // Sends for (UInt i = 0; i < nsend; i++) { if (!sendself || i != self_idx) { buffer &buf = outBuffers[i]; //MPI_Send(&(buf.msize), 1, MPI_INT, buf.pet, tag0, comm); vm.send(static_cast<void*>(&(buf.msize)), sizeof(int), buf.pet, tag0); } } std::vector<ESMCI::VMK::status> stat(enD); if (num_incoming > 0) { for (UInt w = 0; w < enD; w++) { vm.commwait(&commhp[w], &stat[w]); } //ret = MPI_Waitall(enD, &request[0], &status[0]); //if (ret != MPI_SUCCESS) // throw("Bad MPI_WaitAll in setSizes"); } // Now set up true size if (sendself) inSizes[enD] = outBuffers[self_idx].msize; // Third, set up receive sizes if (num_incoming > 0) { inBuffers.resize(num_incoming); inPets.resize(num_incoming); } else { // We need these to have zero sizes since users iterate through them inBuffers.clear(); inPets.clear(); } totalsize = 0; for (UInt i = 0; i < enD; i++) { totalsize += round_to_dword(inSizes[i]); //std::cout << "P:" << rank << ", from " << status[i].MPI_SOURCE << ", size:" << inSizes[i] << std::endl; } recvBuf = new UChar[totalsize+1]; // avoid zero allocation cur_loc = 0; for (UInt i = 0; i < enD; i++) { UInt bsize = round_to_dword(inSizes[i]); inBuffers[i].beg = inBuffers[i].cur = &recvBuf[cur_loc]; inBuffers[i].end = &recvBuf[cur_loc+bsize]; inBuffers[i].bsize = bsize; inBuffers[i].msize = inSizes[i]; inBuffers[i].pet = stat[i].srcPet; petToInBuffer[stat[i].srcPet] = &inBuffers[i]; inPets[i] = stat[i].srcPet; cur_loc += bsize; } // Buffer enD is the special receive buffer from this proc. // Point it straight to the send buffer for this proc. In this // manner, when we copy into the send buffer, the communication is // done; the receive buffer has de facto been filled. if (sendself) { UInt bsize = round_to_dword(inSizes[enD]); inBuffers[enD].beg = inBuffers[enD].cur = outBuffers[self_idx].beg; inBuffers[enD].end = outBuffers[self_idx].end; inBuffers[enD].bsize = bsize; inBuffers[enD].msize = inSizes[enD]; inBuffers[enD].pet = rank; petToInBuffer[rank] = &inBuffers[enD]; inPets[enD] = rank; } // Sort inPets so unpacking can proceed in an expected manner. if (num_incoming > 0) std::sort(inPets.begin(), inPets.end()); for (UInt i = 0; i < num_incoming; i++) delete commhp[i]; obj_state = SIZE; }
void SparseMsg::setSizes(UInt *sizes) { // First, set up send buffers UInt totalsize = 0; // Find total buffer size; round up to word boundaries // for each new buffer for (UInt i = 0; i < nsend; i++) { totalsize += round_to_dword(sizes[i]); } // Allocate send buffer. Delete in case already done delete [] sendBuf; sendBuf = new UChar[totalsize+1]; // Set up pointers into buffer UInt cur_loc = 0; for (UInt i = 0; i < nsend; i++) { UInt bsize = round_to_dword(sizes[i]); buffer &buf = outBuffers[i]; buf.beg = outBuffers[i].cur = &sendBuf[cur_loc]; buf.end = &sendBuf[cur_loc+bsize]; buf.bsize = bsize; buf.msize = sizes[i]; cur_loc += bsize; procToOutBuffer[buf.proc] = &buf; } //std::cout << "last buf end:" << (int) outBuffers[nsend-1].end << ", sendbuf end:" << (int) &sendBuf[cur_loc] << std::endl; // Second, send sizes to receive // avoid allocating zero (add 1) std::vector<MPI_Request> request(num_incoming+1, NULL); std::vector<MPI_Status> status(num_incoming+1); // Post Recieves std::vector<int> inSizes(num_incoming+1); UInt tag0 = 0; UInt enD = num_incoming - (sendself ? 1 : 0); for (UInt i = 0; i < enD; i++) { MPI_Irecv(&inSizes[i], 1, MPI_INT, MPI_ANY_SOURCE, tag0, comm, &request[i]); } // Sends for (UInt i = 0; i < nsend; i++) { if (!sendself || i != self_idx) { buffer &buf = outBuffers[i]; MPI_Send(&(buf.msize), 1, MPI_INT, buf.proc, tag0, comm); } } int ret; if (enD > 0) { ret = MPI_Waitall(enD, &request[0], &status[0]); if (ret != MPI_SUCCESS) throw("Bad MPI_WaitAll in setSizes"); } // Now set up true size if (sendself) inSizes[enD] = outBuffers[self_idx].msize; // Third, set up receive sizes if (num_incoming > 0) { inBuffers.resize(num_incoming); inProcs.resize(num_incoming); } else { // We need these to have zero sizes since users iterate through them inBuffers.clear(); inProcs.clear(); } totalsize = 0; for (UInt i = 0; i < enD; i++) { totalsize += round_to_dword(inSizes[i]); //std::cout << "P:" << rank << ", from " << status[i].MPI_SOURCE << ", size:" << inSizes[i] << std::endl; } delete [] recvBuf; recvBuf = new UChar[totalsize+1]; // avoid zero allocation cur_loc = 0; // Buffer zero is the special receive buffer from this proc. // Point it straight to the send buffer for this proc. The // user won't even know what happened to them. for (UInt i = 0; i < enD; i++) { UInt bsize = round_to_dword(inSizes[i]); inBuffers[i].beg = inBuffers[i].cur = &recvBuf[cur_loc]; inBuffers[i].end = &recvBuf[cur_loc+bsize]; inBuffers[i].bsize = bsize; inBuffers[i].msize = inSizes[i]; inBuffers[i].proc = status[i].MPI_SOURCE; procToInBuffer[status[i].MPI_SOURCE] = &inBuffers[i]; inProcs[i] = status[i].MPI_SOURCE; cur_loc += bsize; } if (sendself) { UInt bsize = round_to_dword(inSizes[enD]); inBuffers[enD].beg = inBuffers[enD].cur = outBuffers[self_idx].beg; inBuffers[enD].end = outBuffers[self_idx].end; inBuffers[enD].bsize = bsize; inBuffers[enD].msize = inSizes[enD]; inBuffers[enD].proc = rank; procToInBuffer[rank] = &inBuffers[enD]; inProcs[enD] = rank; } // Sort inProcs to be conformal with CommRel if (num_incoming > 0) std::sort(inProcs.begin(), inProcs.end()); //std::cout << "last buf end:" << (int) inBuffers[num_incoming-1].end << ", sendbuf end:" << (int) &recvBuf[cur_loc] << std::endl; }
void _fieldStore::Create(UInt nfields, _field **fields, const Attr &attr) { nfield = nfields; // Loop fields; if field lives here, add storage. std::size_t bufSize = 0; // store offsets at beginning of data block offsize = round_to_dword(sizeof(UChar*)*nfields); bufSize += offsize; // Store field dims at beginning of buffer fdimsize = round_to_dword(sizeof(UInt)*nfields); bufSize += fdimsize; // Store field strides at beginning of buffer fstridesize = round_to_dword(sizeof(UInt)*nfields); bufSize += fstridesize; nfreesize = round_to_dword(sizeof(UChar)*NOBJS); bufSize += nfreesize; // first a sizing pass int first_field = -1; int fieldsOnKer = 0; for (UInt f = 0; f < nfields; f++) { _field &fd = *fields[f]; const _fieldLocale *floc; // Get the size of field. Also, make sure we are not in an inconsistent // intersection, i.e. the field dimension is multivalued if ((floc = fd.OnAttr(attr, true))) { fieldsOnKer++; if (first_field < 0) first_field = f; bufSize += round_to_dword(NOBJS*floc->dim*fd.DataSize()); } } // allocate ThrowRequire(data == 0); data = new UChar[bufSize]; totsize = bufSize; UChar *end = data + bufSize; // Pointer to the field data field_data = data + (offsize + fdimsize + fstridesize + nfreesize); std::fill(data, end, (UChar) 0); //std::memset(data,'0',bufSize*sizeof(data[0])); // Set up the offsets buffer, fdim, fstride offsets = reinterpret_cast<UChar**>(data); fdims = reinterpret_cast<UInt*>(&data[offsize]); fstrides = reinterpret_cast<UInt*>(&data[offsize+fdimsize]); nfree = reinterpret_cast<UChar*>(&data[offsize+fdimsize+fstridesize]); UInt dataSize = 0; for (UInt f = 0; f < nfields; f++) { _field &fd = *fields[f]; const _fieldLocale *floc; if ((floc = fd.OnAttr(attr, false))) { offsets[f] = &field_data[dataSize]; dataSize += round_to_dword(NOBJS*floc->dim*fd.DataSize()); fdims[f] = floc->dim; fstrides[f] = floc->dim*fd.DataSize(); } else { offsets[f] = NULL; fdims[f] = 0; fstrides[f] = 0; } } // f // The stride of the next free table; free_stride = sizeof(UChar); // set up next free table for (UChar i = 0; i < NOBJS-1; i++) { nfree[i*free_stride] = i+1; } nfree[(NOBJS-1)*free_stride] = TABLE_FULL; // end of free is_committed = true; }