uint32_t PythonPartitionerDelegator::Partition(void* object, uint32_t partition_number) {
try {
if (_first_round) {
tuple_set_item(_args, 1, boost::python::object(partition_number));
_first_round = false;
}
tuple_set_item(_args, 0, *static_cast<boost::python::object*>(object));
uint32_t ret = boost::python::extract<uint32_t>(_partition(*_args));
return ret;
} catch (boost::python::error_already_set) {
BIGFLOW_HANDLE_PYTHON_EXCEPTION();
} catch (std::exception &e) {
LOG(FATAL) << "Exception: " << e.what()
<< "Calculating partition number error!!!"
<< "Partition number calculated: "
<< boost::python::extract<std::string>(boost::python::str(_partition(*_args)))()
<< "Partition Key: "
<< boost::python::extract<std::string>(tuple_get_item(_args, 0))();
} catch (...) {
LOG(FATAL) << "Calculating partition number error!!!,"
" but I have no idea what happened!";
}
}
void _qsort(int *a, int left, int right) {
if (left<right) {
int pivotIndex = _partition(a, left, right);
_qsort(a, left, pivotIndex-1);
_qsort(a, pivotIndex+1 ,right);
}
}
void _sort(Itr begin, Itr end)
{
if (std::distance(begin, end) <= CUTOFF) return;
boundary_t<Itr> boundary = _partition(begin, end);
_sort(begin, boundary.first);
_sort(boundary.second, end);
}
void _quick_sort(int* nums, int left, int right) {
if (left >= right) {
return;
}
int p = _partition(nums, left, right);
_quick_sort(nums, 0, p-1);
_quick_sort(nums, p+1, right);
}
void qsort(int a[], int left, int right)
{
if( left >= right)
return;
int mid = (left + right)/2;
int pivot = a[mid];
swap(a, right, mid);
int k = _partition(a, pivot, left, right-1);
swap(a, right, k);
qsort(a, left, k-1);
qsort(a, k+1, right);
}
void jqsort(int *arr, int left, int right) {
int pivot, newPivot;
if (left < right) {
pivot = left + ((int) (right-left) / 2);
if (arr[left] > arr[right]) {
if (arr[pivot] > arr[left]) {
pivot = left;
}
} else {
if (arr[pivot] > arr[right]) {
pivot = right;
}
}
newPivot = _partition(arr, left, right, pivot);
jqsort(arr, left, newPivot-1);
jqsort(arr, newPivot+1, right);
}
}
// \smaller contain the first partition (the one with smaller numbers than pivot)
// while \bigger will contain the second one (with bigger numbers)
void partition( Data *data, Data *smaller, Data *bigger )
{
SPD_ASSERT( DAL_isInitialized( smaller ), "smaller Data should be initialized" );
SPD_ASSERT( DAL_isInitialized( bigger ), "bigger Data should be initialized" );
SPD_ASSERT( ! DAL_isInitialized( data ), "data to be partitioned should already contain some data!")
switch( data->medium ) {
case File:
case Array: {
int pivot;
// reading pivot
{
// using a 1-item buffer to load pivot from data
Data buf;
DAL_init( &buf );
SPD_ASSERT( DAL_allocArray( &buf, 1 ), "not enough memory..." );
int pivotIndex = rand() % DAL_dataSize(data);
DAL_dataCopyOS( data, pivotIndex, &buf, 0, 1 ); // reading only pivot
pivot = buf.array.data[ 0 ];
DAL_destroy( &buf );
}
// allocating partitions
DAL_allocData( smaller, DAL_dataSize(data) );
DAL_allocData( bigger, DAL_dataSize(data) );
dal_size_t smallIdx = 0, bigIdx = 0; // items copied into \smaller and \bigger
// initializing buffers
Data bufB, bufS, bufD; // buffers for bigger, smaller and data
dal_size_t bufSize; // size of the three buffers
dal_size_t bbIdx, bsIdx;
{
DAL_init( &bufB );
DAL_init( &bufS );
DAL_init( &bufD );
DAL_allocBuffer( &bufB, DAL_dataSize(data) );
DAL_allocBuffer( &bufS, DAL_dataSize(data) );
DAL_allocBuffer( &bufD, DAL_dataSize(data) );
bufSize = MIN( DAL_dataSize(&bufB), MIN( DAL_dataSize(&bufS), DAL_dataSize(&bufD) ) );
// these should let buffers in memory...
DAL_reallocData( &bufB, bufSize );
DAL_reallocData( &bufS, bufSize );
DAL_reallocData( &bufD, bufSize );
DAL_ASSERT( bufB.medium == Array, &bufB, "Buffer for bigger partition not in memory..." );
DAL_ASSERT( bufS.medium == Array, &bufS, "Buffer for smaller partition not in memory..." );
DAL_ASSERT( bufD.medium == Array, &bufD, "Buffer for data not in memory..." );
}
dal_size_t i = 0, j;
while( i < DAL_dataSize(data) ) {
dal_size_t realBufSize = DAL_dataCopyO( data, i, &bufD, 0 ); // reading one block from \data
// partitioning data in buffers \smaller and \bigger
bbIdx = bsIdx = 0;
for( j = 0; j < realBufSize; ++ j ) {
if( bufD.array.data[j] > pivot ) {
DAL_dataCopyOS( &bufD, j, &bufB, bbIdx, 1 );
bbIdx ++;
} else {
DAL_dataCopyOS( &bufD, j, &bufS, bsIdx, 1 );
bsIdx ++;
}
}
// flushing out \smaller and \bigger
DAL_dataCopyOS( &bufB, 0, bigger, bigIdx, bbIdx );
bigIdx += bbIdx;
DAL_dataCopyOS( &bufS, 0, smaller, smallIdx, bsIdx );
smallIdx += bsIdx;
i += realBufSize;
}
DAL_reallocData( smaller, smallIdx );
DAL_reallocData( bigger, bigIdx );
// destroying buffers
{
DAL_destroy( &bufB );
DAL_destroy( &bufS );
DAL_destroy( &bufD );
}
#if 0
int pivotIndex = rand() % data->array.size;
int pivot = data->array.data[ pivotIndex ];
long lim = _partition( data->array.data, data->array.size, pivot, pivotIndex );
//char buf[128];
//SPD_DEBUG( "partitioned(%d):%s, lim:%ld", pivot, DAL_dataItemsToString(data, buf, sizeof(buf)), lim );
/*
// FIXME
// NOTE: THIS IS NOT SAFE IN GENERAL, SINCE SECOND PARTITION'S DATA WILL GET
// DESTROIED WHEN FIRST PARTITION IS DESTROYED!!!
// BESIDES DATA2 CANNOT BE DESTROYED!!!
// putting second partition in a Data
Data r;
DAL_init( &r );
r.medium = Array;
r.array.data = data->array.data + lim;
r.array.size = data->array.size - lim;
// shrinking first partition
data->array.size = lim;
*/
SPD_ASSERT( DAL_allocArray( bigger, data->array.size - lim ), "not enough memory..." );
memcpy( bigger->array.data, data->array.data + lim, (data->array.size-lim)*sizeof(int) );
SPD_ASSERT( DAL_reallocArray( data, lim ), "wtf, I'm not even growing it here..." );
// moving data from smaller to data
DAL_dataSwap( data, smaller );
#endif
break;
}
default:
DAL_UNSUPPORTED( data );
}
}
