std::vector<int>
mapvector::subsetOfVectorForLoc(size_t id){
std::vector<int> subset(bigVector.begin()+(id*getChunkSize(id-1)),
bigVector.begin()+((id*getChunkSize(id-1))+(getChunkSize(id))));
return subset;
}
qint64 QHttpNetworkReplyPrivate::readReplyBodyChunked(QIODevice *in, QByteDataBuffer *out)
{
qint64 bytes = 0;
while (in->bytesAvailable()) { // while we can read from input
// if we are done with the current chunk, get the size of the new chunk
if (currentChunkRead >= currentChunkSize) {
currentChunkSize = 0;
currentChunkRead = 0;
if (bytes) {
char crlf[2];
bytes += in->read(crlf, 2); // read the "\r\n" after the chunk
}
bytes += getChunkSize(in, ¤tChunkSize);
if (currentChunkSize == -1)
break;
}
// if the chunk size is 0, end of the stream
if (currentChunkSize == 0) {
state = AllDoneState;
break;
}
// otherwise, try to read what is missing for this chunk
qint64 haveRead = readReplyBodyRaw (in, out, currentChunkSize - currentChunkRead);
currentChunkRead += haveRead;
bytes += haveRead;
// ### error checking here
}
return bytes;
}
END_TEST
START_TEST(test_getChunkSize)
{
printf("Get Chunk Size Test(s)\n");
const int num_tests = 5;
enum NUMERALS test_cases[5][5];
stringToEnum("MCM", test_cases[0]);
stringToEnum("IX", test_cases[1]);
stringToEnum("IV", test_cases[2]);
stringToEnum("MC", test_cases[3]);
stringToEnum("CL", test_cases[4]);
const size_t lens[5] = {
3,
2,
2,
2,
2
};
const int isValidCase[] = {
1,
2,
2,
1,
1
};
int i;
for(i=0; i<num_tests; i++){
ck_assert_int_eq(getChunkSize(test_cases[i], lens[i]), isValidCase[i]);
}
}
Status SettingsType::validate() const {
if (!_key.is_initialized() || _key->empty()) {
return Status(ErrorCodes::NoSuchKey,
str::stream() << "missing " << key.name() << " field");
}
if (_key == ChunkSizeDocKey) {
if (!(getChunkSize() > 0)) {
return Status(ErrorCodes::BadValue,
str::stream() << "chunksize specified in " << chunkSize.name()
<< " field must be greater than zero");
}
}
else if (_key == BalancerDocKey) {
if (_secondaryThrottle.is_initialized() &&
_migrationWriteConcern.is_initialized()) {
return Status(ErrorCodes::BadValue,
str::stream() << "cannot have both secondary throttle and "
<< "migration write concern set at the same time");
}
}
else {
return Status(ErrorCodes::UnsupportedFormat,
str::stream() << "unsupported key in " << key.name() << " field");
}
return Status::OK();
}
ChunkAllocator(size_t chunkSize) :
m_ChunkSize(chunkSize), m_AvailableSlots(ChunkAllocatorBlocks), m_pData((char*) malloc(getAllocationSize(chunkSize))) {
assert(PAGE_SIZE == (size_t) getPageSize());
memset(m_UsedSlot, false, ChunkAllocatorBlocks);
char* pBlockStart = reinterpret_cast<char*>(upperPage(reinterpret_cast<size_t>(m_pData)));
const size_t realChunkSize = getChunkSize(chunkSize);
for (int i = 0; i < ChunkAllocatorBlocks; ++i) {
m_SlotData[i] = pBlockStart;
Sentinel *pSentinel = getSentinelFromBlockStart(pBlockStart);
pSentinel->pAllocator = this;
pSentinel->magic = 0xDEADC0DE;
pBlockStart += realChunkSize;
}
}
BSONObj SettingsType::toBSON() const {
BSONObjBuilder builder;
if (_key) builder.append(key(), getKey());
if (_chunkSize) builder.append(chunkSize(), getChunkSize());
if (_balancerStopped) builder.append(balancerStopped(), getBalancerStopped());
if (_secondaryThrottle) {
builder.append(deprecated_secondaryThrottle(), getSecondaryThrottle());
}
if (_migrationWriteConcern) {
builder.append(migrationWriteConcern(), getMigrationWriteConcern().toBSON());
}
if (_waitForDelete) builder.append(waitForDelete(), getWaitForDelete());
return builder.obj();
}
qint64 QHttpNetworkReplyPrivate::readReplyBodyChunked(QAbstractSocket *socket, QByteDataBuffer *out)
{
qint64 bytes = 0;
while (socket->bytesAvailable()) {
if (currentChunkRead >= currentChunkSize) {
// For the first chunk and when we're done with a chunk
currentChunkSize = 0;
currentChunkRead = 0;
if (bytes) {
// After a chunk
char crlf[2];
// read the "\r\n" after the chunk
qint64 haveRead = socket->read(crlf, 2);
// FIXME: This code is slightly broken and not optimal. What if the 2 bytes are not available yet?!
// For nice reasons (the toLong in getChunkSize accepting \n at the beginning
// it right now still works, but we should definitely fix this.
if (haveRead != 2)
return bytes; // FIXME
bytes += haveRead;
}
// Note that chunk size gets stored in currentChunkSize, what is returned is the bytes read
bytes += getChunkSize(socket, ¤tChunkSize);
if (currentChunkSize == -1)
break;
}
// if the chunk size is 0, end of the stream
if (currentChunkSize == 0) {
state = AllDoneState;
break;
}
// otherwise, try to begin reading this chunk / to read what is missing for this chunk
qint64 haveRead = readReplyBodyRaw (socket, out, currentChunkSize - currentChunkRead);
currentChunkRead += haveRead;
bytes += haveRead;
// ### error checking here
}
return bytes;
}
qint64 QHttpNetworkReplyPrivate::readReplyBodyChunked(QAbstractSocket *socket, QByteDataBuffer *out)
{
qint64 bytes = 0;
while (socket->bytesAvailable()) {
if (readBufferMaxSize && (bytes > readBufferMaxSize))
break;
if (!lastChunkRead && currentChunkRead >= currentChunkSize) {
// For the first chunk and when we're done with a chunk
currentChunkSize = 0;
currentChunkRead = 0;
if (bytes) {
// After a chunk
char crlf[2];
// read the "\r\n" after the chunk
qint64 haveRead = socket->read(crlf, 2);
// FIXME: This code is slightly broken and not optimal. What if the 2 bytes are not available yet?!
// For nice reasons (the toLong in getChunkSize accepting \n at the beginning
// it right now still works, but we should definitely fix this.
if (haveRead != 2)
return bytes; // FIXME
bytes += haveRead;
}
// Note that chunk size gets stored in currentChunkSize, what is returned is the bytes read
bytes += getChunkSize(socket, ¤tChunkSize);
if (currentChunkSize == -1)
break;
}
// if the chunk size is 0, end of the stream
if (currentChunkSize == 0 || lastChunkRead) {
lastChunkRead = true;
// try to read the "\r\n" after the chunk
char crlf[2];
qint64 haveRead = socket->read(crlf, 2);
if (haveRead > 0)
bytes += haveRead;
if ((haveRead == 2 && crlf[0] == '\r' && crlf[1] == '\n') || (haveRead == 1 && crlf[0] == '\n'))
state = AllDoneState;
else if (haveRead == 1 && crlf[0] == '\r')
break; // Still waiting for the last \n
else if (haveRead > 0) {
// If we read something else then CRLF, we need to close the channel.
forceConnectionCloseEnabled = true;
state = AllDoneState;
}
break;
}
// otherwise, try to begin reading this chunk / to read what is missing for this chunk
qint64 haveRead = readReplyBodyRaw (socket, out, currentChunkSize - currentChunkRead);
currentChunkRead += haveRead;
bytes += haveRead;
// ### error checking here
}
return bytes;
}
inline static size_t getAllocationSize(size_t chunkSize) {
const size_t allAlignedChunkSize = getChunkSize(chunkSize) * ChunkAllocatorBlocks;
// adding one PAGE_SIZE to ensure we can offset and give aligned data
return allAlignedChunkSize + PAGE_SIZE;
}
uint32_t TankFile::FileEntry::getChunkIndex(const uint32_t offs) const
{
assert(getChunkSize() != 0);
return offs / getChunkSize();
}
/**
* constructor
*/
SellCSigma_Matrix::SellCSigma_Matrix( MMreader mmMatrix, int C, int const sigma )
:C_(C), sigma_(sigma)
,M_(mmMatrix.getRows()), N_(mmMatrix.getCols())
,nz_(mmMatrix.getNonZeros()), numberOfChunks_((M_-1)/C+1)
,colInd_(nullptr)
,chunkPtr_(new int[numberOfChunks_]), chunkLength_(new int[numberOfChunks_])
,permute_(new int[M_]), antiPermute_(new int[M_])
,val_(nullptr)
{
// sort input Matrix by row ID
if( !mmMatrix.isRowSorted() )
sortByRow(mmMatrix);
std::vector< std::tuple<int,int,double> > & mmData = mmMatrix.getMatrx();
std::vector< std::tuple<int, int> > rowLengths = getRowLengths(mmMatrix);
// sort sigam chunks by row length
auto begin = rowLengths.begin();
auto end = rowLengths.begin() + getSigma();
for (; end <= rowLengths.end(); begin += getSigma(), end += getSigma() )
{
std::sort(begin, end,
[](std::tuple<int,int> const & a, std::tuple<int,int> const & b)
{return std::get<1>(a) < std::get<1>(b);}
);
}
begin -= getSigma();
std::sort(begin, rowLengths.end(),
[](std::tuple<int,int> const & a, std::tuple<int,int> const & b)
{return std::get<1>(a) < std::get<1>(b);}
);
// determine chunk length and size
// and set backword permutation
std::vector<int> valuesPerChunk( getNumberOfChunks() );
#ifdef _OPENMP
#pragma omp parallel for schedule(runtime)
#endif
for (int chunk=0; chunk < getNumberOfChunks(); ++chunk)
{
int maxRowLenghth = 0;
for (int i=0, row=chunk*getChunkSize();
i<getChunkSize() && row<getRows();
++i, ++row
)
{
if ( maxRowLenghth < std::get<1>(rowLengths[row]) )
maxRowLenghth = std::get<1>(rowLengths[row]);
// set backword permutation
antiPermute_[ std::get<0>(rowLengths[row]) ] = row;
}
chunkLength_[chunk] = maxRowLenghth;
valuesPerChunk[chunk] = maxRowLenghth * getChunkSize();
}
// calculate memory usage and allocate memmory for values and colum IDs
capasety_ = std::accumulate(std::begin(valuesPerChunk),
std::end(valuesPerChunk),
0
);
val_ = new double[capasety_];
colInd_ = new int [capasety_];
// calulate memory overhead
overhead_ = capasety_ - getNonZeros();
// creat Sell-C-sigma data
std::vector<int> chunkOffset = getOffsets(valuesPerChunk);
std::vector<int> rowOffset = getOffsets(getValsPerRow(mmMatrix));
#ifdef _OPENMP
#pragma omp parallel for schedule(runtime)
#endif
for (int chunk=0; chunk < getNumberOfChunks(); ++chunk)
{
chunkPtr_[chunk] = chunkOffset[chunk];
for (int j=0; j<chunkLength_[chunk]; ++j)
{
for (int i=0, row=chunk*getChunkSize();
i<getChunkSize();
++i, ++row
)
{
int col;
double val;
if (row<getRows())
{
// set permutation
permute_[row] = std::get<0>(rowLengths[row]);
// finde values and collumn index
if ( j < std::get<1>(rowLengths[row]) )
{ // fill with matrix values
int id = rowOffset[ permute_[row] ] + j;
val = std::get<2>( mmData[id] );
col = std::get<1>( mmData[id] );
}
else
{ // fill chunk with 0
val = 0.;
col = 0;
}
}
else
{ // add zero rows to end of matrix fill up last chunk
val = 0.;
col = 0;
}
val_ [chunkPtr_[chunk] + i + j*getChunkSize()] = val;
colInd_[chunkPtr_[chunk] + i + j*getChunkSize()] = antiPermute_[col];
}
}
}
/*
std::cout << "Sell-C-sigma constructed:"
<< "\nC: " << getChunkSize() << " sigma: " << getSigma()
<< "\n(" << getRows() << "," << getCols() << ") " << getNonZeros()
<< ":\n";
for (int i=0; i<valueMemoryUsage; ++i)
{
std::cout << getValues()[i] << " (" << getColInd()[i] << ")\n";
}
std::cout << std::endl;
*/
}
dp::rix::fx::SmartChunk BufferManagerIndexed::allocateChunk()
{
return new Chunk( this, getAlignedBlockSize(), getChunkSize() );
}
