ColumnPtr ColumnArray::indexImpl(const PaddedPODArray<T> & indexes, size_t limit) const
{
if (limit == 0)
return ColumnArray::create(data);
/// Convert indexes to UInt64 in case of overflow.
auto nested_indexes_column = ColumnUInt64::create();
PaddedPODArray<UInt64> & nested_indexes = nested_indexes_column->getData();
nested_indexes.reserve(getOffsets().back());
auto res = ColumnArray::create(data->cloneEmpty());
Offsets & res_offsets = res->getOffsets();
res_offsets.resize(limit);
size_t current_offset = 0;
for (size_t i = 0; i < limit; ++i)
{
for (size_t j = 0; j < sizeAt(indexes[i]); ++j)
nested_indexes.push_back(offsetAt(indexes[i]) + j);
current_offset += sizeAt(indexes[i]);
res_offsets[i] = current_offset;
}
if (current_offset != 0)
res->data = data->index(*nested_indexes_column, current_offset);
return res;
}
void ColumnArray::insertDefault()
{
/// NOTE 1: We can use back() even if the array is empty (due to zero -1th element in PODArray).
/// NOTE 2: We cannot use reference in push_back, because reference get invalidated if array is reallocated.
auto last_offset = getOffsets().back();
getOffsets().push_back(last_offset);
}
void ColumnArray::insert(const Field & x)
{
const Array & array = DB::get<const Array &>(x);
size_t size = array.size();
for (size_t i = 0; i < size; ++i)
getData().insert(array[i]);
getOffsets().push_back(getOffsets().back() + size);
}
void ColumnArray::insertFrom(const IColumn & src_, size_t n)
{
const ColumnArray & src = static_cast<const ColumnArray &>(src_);
size_t size = src.sizeAt(n);
size_t offset = src.offsetAt(n);
getData().insertRangeFrom(src.getData(), offset, size);
getOffsets().push_back(getOffsets().back() + size);
}
const char * ColumnArray::deserializeAndInsertFromArena(const char * pos)
{
size_t array_size = unalignedLoad<size_t>(pos);
pos += sizeof(array_size);
for (size_t i = 0; i < array_size; ++i)
pos = getData().deserializeAndInsertFromArena(pos);
getOffsets().push_back(getOffsets().back() + array_size);
return pos;
}
ColumnPtr ColumnArray::filterNumber(const Filter & filt, ssize_t result_size_hint) const
{
if (getOffsets().size() == 0)
return std::make_shared<ColumnArray>(data);
auto res = std::make_shared<ColumnArray>(data->cloneEmpty());
auto & res_elems = static_cast<ColumnVector<T> &>(res->getData()).getData();
Offsets_t & res_offsets = res->getOffsets();
filterArraysImpl<T>(static_cast<const ColumnVector<T> &>(*data).getData(), getOffsets(), res_elems, res_offsets, filt, result_size_hint);
return res;
}
//this function will print a samfile from the bamfile
int motherView(bufReader *rd,int nFiles,std::vector<regs>regions) {
aRead b;
b.vDat=new uint8_t[RLEN];
kstring_t str; str.s=NULL; str.l=str.m=0;
if(regions.size()==0) {//print all
int block_size;
while(SIG_COND && bgzf_read(rd[0].fp,&block_size,sizeof(int))){
getAlign(rd[0].fp,block_size,b);
printReadBuffered(b,rd[0].hd,str);
fprintf(stdout,"%s",str.s);
}
}else {
for(int i=0;i<(int)regions.size();i++){
int tmpRef = regions[i].refID;
int tmpStart = regions[i].start;
int tmpStop = regions[i].stop;
getOffsets(rd[0].bamfname,rd[0].hd,rd[0].it,tmpRef,tmpStart,tmpStop);
int ret =0;
while(SIG_COND){
ret = bam_iter_read(rd[0].fp, &rd[0].it, b);
if(ret<0)
break;
printReadBuffered(b,rd[0].hd,str);
fprintf(stdout,"%s",str.s);
}
free(rd[0].it.off);//the offsets
}
free(str.s);
delete [] b.vDat;
}
return 0;
}
void ColumnArray::insertRangeFrom(const IColumn & src, size_t start, size_t length)
{
if (length == 0)
return;
const ColumnArray & src_concrete = static_cast<const ColumnArray &>(src);
if (start + length > src_concrete.getOffsets().size())
throw Exception("Parameter out of bound in ColumnArray::insertRangeFrom method.",
ErrorCodes::PARAMETER_OUT_OF_BOUND);
size_t nested_offset = src_concrete.offsetAt(start);
size_t nested_length = src_concrete.getOffsets()[start + length - 1] - nested_offset;
data->insertRangeFrom(src_concrete.getData(), nested_offset, nested_length);
Offsets_t & cur_offsets = getOffsets();
const Offsets_t & src_offsets = src_concrete.getOffsets();
if (start == 0 && cur_offsets.empty())
{
cur_offsets.assign(src_offsets.begin(), src_offsets.begin() + length);
}
else
{
size_t old_size = cur_offsets.size();
size_t prev_max_offset = old_size ? cur_offsets.back() : 0;
cur_offsets.resize(old_size + length);
for (size_t i = 0; i < length; ++i)
cur_offsets[old_size + i] = src_offsets[start + i] - nested_offset + prev_max_offset;
}
}
Eigen::VectorXd& BCIDataSet::getInstance(int i)
{
int epoch = 0, t0 = 0;
getOffsets(i, epoch, t0);
buildInstance(epoch, t0);
return tempInstance;
}
ColumnPtr ColumnArray::filterTuple(const Filter & filt, ssize_t result_size_hint) const
{
if (getOffsets().size() == 0)
return ColumnArray::create(data);
const ColumnTuple & tuple = static_cast<const ColumnTuple &>(*data);
/// Make temporary arrays for each components of Tuple, then filter and collect back.
size_t tuple_size = tuple.getColumns().size();
if (tuple_size == 0)
throw Exception("Logical error: empty tuple", ErrorCodes::LOGICAL_ERROR);
Columns temporary_arrays(tuple_size);
for (size_t i = 0; i < tuple_size; ++i)
temporary_arrays[i] = ColumnArray(tuple.getColumns()[i]->assumeMutable(), getOffsetsPtr()->assumeMutable())
.filter(filt, result_size_hint);
Columns tuple_columns(tuple_size);
for (size_t i = 0; i < tuple_size; ++i)
tuple_columns[i] = static_cast<const ColumnArray &>(*temporary_arrays[i]).getDataPtr();
return ColumnArray::create(
ColumnTuple::create(tuple_columns),
static_cast<const ColumnArray &>(*temporary_arrays.front()).getOffsetsPtr());
}
/* Median filter: pixel goes with a majority of neighboring pixel */
void medianify(unsigned char *mask, unsigned char *median, int rows, int cols) {
long imagesize, i;
int j, total;
int *offsets;
imagesize = rows * cols;
offsets = getOffsets(cols, imagesize);
for (i=0; i < imagesize; ++i){ /* for each pixel */
/* don't process the 1 pixel layer around the whole image */
if (edge(i, cols, imagesize)) median[i] = mask[i];
else {
total = 0;
/* count number of foreground pixels in i and neighbors, set i to majority */
for (j = 0; j < 9; ++j) {
if ( mask[offsets[j] + i] == FOREGROUND) {
++total;
}
}
if (total > 4) {
median[i] = FOREGROUND;
}
else {
median[i] = BACKGROUND;
}
}
}
}
void ColumnArray::popBack(size_t n)
{
auto & offsets_data = getOffsets();
size_t nested_n = offsets_data.back() - offsetAt(offsets_data.size() - n);
if (nested_n)
getData().popBack(nested_n);
offsets_data.resize_assume_reserved(offsets_data.size() - n);
}
ColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hint) const
{
size_t size = getOffsets().size();
if (size != filt.size())
throw Exception("Size of filter doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
if (size == 0)
return std::make_shared<ColumnArray>(data);
Filter nested_filt(getOffsets().back());
for (size_t i = 0; i < size; ++i)
{
if (filt[i])
memset(&nested_filt[offsetAt(i)], 1, sizeAt(i));
else
memset(&nested_filt[offsetAt(i)], 0, sizeAt(i));
}
std::shared_ptr<ColumnArray> res = std::make_shared<ColumnArray>(data);
ssize_t nested_result_size_hint = 0;
if (result_size_hint < 0)
nested_result_size_hint = result_size_hint;
else if (result_size_hint && result_size_hint < 1000000000 && data->size() < 1000000000) /// Избегаем переполнения.
nested_result_size_hint = result_size_hint * data->size() / size;
res->data = data->filter(nested_filt, nested_result_size_hint);
Offsets_t & res_offsets = res->getOffsets();
if (result_size_hint)
res_offsets.reserve(result_size_hint > 0 ? result_size_hint : size);
size_t current_offset = 0;
for (size_t i = 0; i < size; ++i)
{
if (filt[i])
{
current_offset += sizeAt(i);
res_offsets.push_back(current_offset);
}
}
return res;
}
/// Put implementation here to avoid extra linking dependencies for clickhouse_common_io
void dumpToArrayColumns(const Counters & counters, DB::IColumn * column_names_, DB::IColumn * column_values_, bool nonzero_only)
{
/// Convert ptr and make simple check
auto column_names = (column_names_) ? &typeid_cast<DB::ColumnArray &>(*column_names_) : nullptr;
auto column_values = (column_values_) ? &typeid_cast<DB::ColumnArray &>(*column_values_) : nullptr;
size_t size = 0;
for (Event event = 0; event < Counters::num_counters; ++event)
{
UInt64 value = counters[event].load(std::memory_order_relaxed);
if (nonzero_only && 0 == value)
continue;
++size;
if (column_names)
{
const char * desc = ProfileEvents::getName(event);
column_names->getData().insertData(desc, strlen(desc));
}
if (column_values)
column_values->getData().insert(value);
}
if (column_names)
{
auto & offsets = column_names->getOffsets();
offsets.push_back(offsets.back() + size);
}
if (column_values)
{
/// Nested columns case
bool the_same_offsets = column_names && column_names->getOffsetsPtr().get() == column_values->getOffsetsPtr().get();
if (!the_same_offsets)
{
auto & offsets = column_values->getOffsets();
offsets.push_back(offsets.back() + size);
}
}
}
ColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hint) const
{
size_t size = getOffsets().size();
if (size != filt.size())
throw Exception("Size of filter doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
if (size == 0)
return ColumnArray::create(data);
Filter nested_filt(getOffsets().back());
for (size_t i = 0; i < size; ++i)
{
if (filt[i])
memset(&nested_filt[offsetAt(i)], 1, sizeAt(i));
else
memset(&nested_filt[offsetAt(i)], 0, sizeAt(i));
}
auto res = ColumnArray::create(data->cloneEmpty());
ssize_t nested_result_size_hint = 0;
if (result_size_hint < 0)
nested_result_size_hint = result_size_hint;
else if (result_size_hint && result_size_hint < 1000000000 && data->size() < 1000000000) /// Avoid overflow.
nested_result_size_hint = result_size_hint * data->size() / size;
res->data = data->filter(nested_filt, nested_result_size_hint);
Offsets & res_offsets = res->getOffsets();
if (result_size_hint)
res_offsets.reserve(result_size_hint > 0 ? result_size_hint : size);
size_t current_offset = 0;
for (size_t i = 0; i < size; ++i)
{
if (filt[i])
{
current_offset += sizeAt(i);
res_offsets.push_back(current_offset);
}
}
return res;
}
void MatrixOpData::cleanUp(double offsetScale)
{
const ArrayDouble & a = getArray();
const ArrayDouble::Values & m = a.getValues();
const unsigned long dim = a.getLength();
// Estimate the magnitude of the matrix.
double max_val = 0.;
for (unsigned long i = 0; i<dim; ++i)
{
for (unsigned long j = 0; j<dim; ++j)
{
const double val = fabs(m[i * dim + j]);
max_val = max_val > val ? max_val : val;
}
}
// Determine an absolute tolerance.
// TODO: For double matrices a smaller tolerance could be used. However we
// have matrices that may have been quantized to less than double precision
// either from being written to files or via the factories that take float
// args. In any case, the tolerance is small enough to pick up anything
// that would be significant in the context of color management.
const double scale = max_val > 1e-4 ? max_val : 1e-4;
const double abs_tol = scale * 1e-6;
// Replace values that are close to integers by exact values.
for (unsigned long i = 0; i<dim; ++i)
{
for (unsigned long j = 0; j<dim; ++j)
{
const double val = m[i * dim + j];
const double round_val = round(val);
const double diff = fabs(val - round_val);
if (diff < abs_tol)
{
setArrayValue(i * dim + j, round_val);
}
}
}
// Do likewise for the offsets.
const double scale2 = offsetScale > 1e-4 ? offsetScale : 1e-4;
const double abs_tol2 = scale2 * 1e-6;
for (unsigned long i = 0; i<dim; ++i)
{
const double val = getOffsets()[i];
const double round_val = round(val);
const double diff = fabs(val - round_val);
if (diff < abs_tol2)
{
setOffsetValue(i, round_val);
}
}
}
bool ColumnArray::hasEqualOffsets(const ColumnArray & other) const
{
if (offsets == other.offsets)
return true;
const Offsets & offsets1 = getOffsets();
const Offsets & offsets2 = other.getOffsets();
return offsets1.size() == offsets2.size()
&& (offsets1.size() == 0 || 0 == memcmp(offsets1.data(), offsets2.data(), sizeof(offsets1[0]) * offsets1.size()));
}
void ColumnArray::insertData(const char * pos, size_t length)
{
/** Similarly - only for arrays of fixed length values.
*/
IColumn * data_ = &getData();
if (!data_->isFixedAndContiguous())
throw Exception("Method insertData is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
size_t field_size = data_->sizeOfValueIfFixed();
const char * end = pos + length;
size_t elems = 0;
for (; pos + field_size <= end; pos += field_size, ++elems)
data_->insertData(pos, field_size);
if (pos != end)
throw Exception("Incorrect length argument for method ColumnArray::insertData", ErrorCodes::BAD_ARGUMENTS);
getOffsets().push_back(getOffsets().back() + elems);
}
ColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const
{
if (getOffsets().size() == 0)
return ColumnArray::create(data);
const ColumnNullable & nullable_elems = static_cast<const ColumnNullable &>(*data);
auto array_of_nested = ColumnArray::create(nullable_elems.getNestedColumnPtr(), offsets);
auto filtered_array_of_nested_owner = array_of_nested->filter(filt, result_size_hint);
auto & filtered_array_of_nested = static_cast<const ColumnArray &>(*filtered_array_of_nested_owner);
auto & filtered_offsets = filtered_array_of_nested.getOffsetsPtr();
auto res_null_map = ColumnUInt8::create();
filterArraysImplOnlyData(nullable_elems.getNullMapData(), getOffsets(), res_null_map->getData(), filt, result_size_hint);
return ColumnArray::create(
ColumnNullable::create(
filtered_array_of_nested.getDataPtr(),
std::move(res_null_map)),
filtered_offsets);
}
ColumnPtr ColumnArray::replicateNumber(const Offsets & replicate_offsets) const
{
size_t col_size = size();
if (col_size != replicate_offsets.size())
throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
MutableColumnPtr res = cloneEmpty();
if (0 == col_size)
return res;
ColumnArray & res_ = typeid_cast<ColumnArray &>(*res);
const typename ColumnVector<T>::Container & src_data = typeid_cast<const ColumnVector<T> &>(*data).getData();
const Offsets & src_offsets = getOffsets();
typename ColumnVector<T>::Container & res_data = typeid_cast<ColumnVector<T> &>(res_.getData()).getData();
Offsets & res_offsets = res_.getOffsets();
res_data.reserve(data->size() / col_size * replicate_offsets.back());
res_offsets.reserve(replicate_offsets.back());
Offset prev_replicate_offset = 0;
Offset prev_data_offset = 0;
Offset current_new_offset = 0;
for (size_t i = 0; i < col_size; ++i)
{
size_t size_to_replicate = replicate_offsets[i] - prev_replicate_offset;
size_t value_size = src_offsets[i] - prev_data_offset;
for (size_t j = 0; j < size_to_replicate; ++j)
{
current_new_offset += value_size;
res_offsets.push_back(current_new_offset);
if (value_size)
{
res_data.resize(res_data.size() + value_size);
memcpy(&res_data[res_data.size() - value_size], &src_data[prev_data_offset], value_size * sizeof(T));
}
}
prev_replicate_offset = replicate_offsets[i];
prev_data_offset = src_offsets[i];
}
return res;
}
ColumnPtr ColumnArray::permute(const Permutation & perm, size_t limit) const
{
size_t size = getOffsets().size();
if (limit == 0)
limit = size;
else
limit = std::min(size, limit);
if (perm.size() < limit)
throw Exception("Size of permutation is less than required.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
if (limit == 0)
return std::make_shared<ColumnArray>(data);
Permutation nested_perm(getOffsets().back());
std::shared_ptr<ColumnArray> res = std::make_shared<ColumnArray>(data->cloneEmpty());
Offsets_t & res_offsets = res->getOffsets();
res_offsets.resize(limit);
size_t current_offset = 0;
for (size_t i = 0; i < limit; ++i)
{
for (size_t j = 0; j < sizeAt(perm[i]); ++j)
nested_perm[current_offset + j] = offsetAt(perm[i]) + j;
current_offset += sizeAt(perm[i]);
res_offsets[i] = current_offset;
}
if (current_offset != 0)
res->data = data->permute(nested_perm, current_offset);
return res;
}
template <> ColumnPtr ColumnConst<String>::convertToFullColumn() const
{
if (!data_type || typeid_cast<const DataTypeString *>(&*data_type))
{
auto res = std::make_shared<ColumnString>();
ColumnString::Offsets_t & offsets = res->getOffsets();
ColumnString::Chars_t & vec = res->getChars();
size_t string_size = data.size() + 1;
size_t offset = 0;
offsets.resize(s);
vec.resize(s * string_size);
for (size_t i = 0; i < s; ++i)
{
memcpy(&vec[offset], data.data(), string_size);
offset += string_size;
offsets[i] = offset;
}
return res;
}
else if (const DataTypeFixedString * type = typeid_cast<const DataTypeFixedString *>(&*data_type))
{
size_t n = type->getN();
if (data.size() > n)
throw Exception("Too long value for " + type->getName(), ErrorCodes::TOO_LARGE_STRING_SIZE);
auto res = std::make_shared<ColumnFixedString>(n);
ColumnFixedString::Chars_t & vec = res->getChars();
vec.resize_fill(n * s);
size_t offset = 0;
for (size_t i = 0; i < s; ++i)
{
memcpy(&vec[offset], data.data(), data.size());
offset += n;
}
return res;
}
else
throw Exception("Invalid data type in ColumnConstString: " + data_type->getName(), ErrorCodes::LOGICAL_ERROR);
}
StringRef ColumnArray::getDataAt(size_t n) const
{
/** Returns the range of memory that covers all elements of the array.
* Works for arrays of fixed length values.
* For arrays of strings and arrays of arrays, the resulting chunk of memory may not be one-to-one correspondence with the elements,
* since it contains only the data laid in succession, but not the offsets.
*/
size_t offset_of_first_elem = offsetAt(n);
StringRef first = getData().getDataAtWithTerminatingZero(offset_of_first_elem);
size_t array_size = sizeAt(n);
if (array_size == 0)
return StringRef(first.data, 0);
size_t offset_of_last_elem = getOffsets()[n] - 1;
StringRef last = getData().getDataAtWithTerminatingZero(offset_of_last_elem);
return StringRef(first.data, last.data + last.size - first.data);
}
void MatrixOpData::finalize()
{
AutoMutex lock(m_mutex);
std::ostringstream cacheIDStream;
cacheIDStream << getID();
md5_state_t state;
md5_byte_t digest[16];
// TODO: array and offset do not require double precison in cache.
md5_init(&state);
md5_append(&state,
(const md5_byte_t *)&(getArray().getValues()[0]),
(int)(16 * sizeof(double)));
md5_append(&state,
(const md5_byte_t *)getOffsets().getValues(),
(int)(4 * sizeof(double)));
md5_finish(&state, digest);
cacheIDStream << GetPrintableHash(digest);
m_cacheID = cacheIDStream.str();
}
ColumnPtr ColumnConst<Array>::convertToFullColumn() const
{
if (!data_type)
throw Exception("No data type specified for ColumnConstArray", ErrorCodes::LOGICAL_ERROR);
const DataTypeArray * type = typeid_cast<const DataTypeArray *>(&*data_type);
if (!type)
throw Exception("Non-array data type specified for ColumnConstArray", ErrorCodes::LOGICAL_ERROR);
const Array & array = getDataFromHolderImpl();
size_t array_size = array.size();
const auto & nested_type = type->getNestedType();
ColumnPtr nested_column;
if (nested_type->isNull())
{
/// Special case: an array of Null is actually an array of Nullable(UInt8).
nested_column = std::make_shared<ColumnNullable>(
std::make_shared<ColumnUInt8>(), std::make_shared<ColumnUInt8>());
}
else
nested_column = type->getNestedType()->createColumn();
auto res = std::make_shared<ColumnArray>(nested_column);
ColumnArray::Offsets_t & offsets = res->getOffsets();
offsets.resize(s);
for (size_t i = 0; i < s; ++i)
{
offsets[i] = (i + 1) * array_size;
for (size_t j = 0; j < array_size; ++j)
nested_column->insert(array[j]);
}
return res;
}
/* turn off any foreground pixel with a background pixel neighbor (8-connected) */
void shrink(unsigned char *mask, unsigned char *shrunk, int rows, int cols) {
long imagesize, i;
int j;
int *offsets;
imagesize = rows * cols;
offsets = getOffsets(cols, imagesize);
for (i=0; i < imagesize; ++i){
/* don't process the 1 pixel layer around the whole image */
if (edge(i, cols, imagesize)) shrunk[i] = mask[i];
/* ignore if background */
else if (mask[i] == BACKGROUND) {
shrunk[i] = BACKGROUND;
}
/* if foreground, apply shrinking algorithm */
else if (mask[i] == FOREGROUND) {
shrunk[i] = FOREGROUND;
/* look at 8 nearest neighbors. If ANY are foreground, turn pixel ON */
for (j = 0; j < 9; ++j) {
if( mask[ i + offsets[j]] == BACKGROUND){
shrunk[i] = BACKGROUND;
break;
}
}
}
else {
printf("Error: Hit a value that's not 0 or 255!\n");
shrunk[i] = mask[i];
}
}
}
MatrixOpDataRcPtr MatrixOpData::inverse() const
{
// Get the inverse matrix.
MatrixArrayPtr invMatrixArray = m_array.inverse();
// MatrixArray::inverse() will throw for singular matrices.
// Calculate the inverse offset.
const Offsets& offsets = getOffsets();
Offsets invOffsets;
if (offsets.isNotNull())
{
invMatrixArray->inner(offsets, invOffsets);
invOffsets.scale(-1);
}
MatrixOpDataRcPtr invOp = std::make_shared<MatrixOpData>(getOutputBitDepth(), getInputBitDepth());
invOp->setRGBA(&(invMatrixArray->getValues()[0]));
invOp->setOffsets(invOffsets);
// No need to call validate(), the invOp will have proper dimension,
// bit-depths, matrix and offets values.
return invOp;
}
ColumnPtr ColumnArray::replicateConst(const Offsets_t & replicate_offsets) const
{
size_t col_size = size();
if (col_size != replicate_offsets.size())
throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
if (0 == col_size)
return cloneEmpty();
const Offsets_t & src_offsets = getOffsets();
auto res_column_offsets = std::make_shared<ColumnOffsets_t>();
Offsets_t & res_offsets = res_column_offsets->getData();
res_offsets.reserve(replicate_offsets.back());
Offset_t prev_replicate_offset = 0;
Offset_t prev_data_offset = 0;
Offset_t current_new_offset = 0;
for (size_t i = 0; i < col_size; ++i)
{
size_t size_to_replicate = replicate_offsets[i] - prev_replicate_offset;
size_t value_size = src_offsets[i] - prev_data_offset;
for (size_t j = 0; j < size_to_replicate; ++j)
{
current_new_offset += value_size;
res_offsets.push_back(current_new_offset);
}
prev_replicate_offset = replicate_offsets[i];
prev_data_offset = src_offsets[i];
}
return std::make_shared<ColumnArray>(getData().cloneResized(current_new_offset), res_column_offsets);
}
void execute(Block & block, const ColumnNumbers & arguments, size_t result) override
{
const ColumnWithTypeAndName & arg_from = block.unsafeGetByPosition(arguments[0]);
const ColumnWithTypeAndName & arg_charset_from = block.unsafeGetByPosition(arguments[1]);
const ColumnWithTypeAndName & arg_charset_to = block.unsafeGetByPosition(arguments[2]);
ColumnWithTypeAndName & res = block.unsafeGetByPosition(result);
const ColumnConstString * col_charset_from = typeid_cast<const ColumnConstString *>(arg_charset_from.column.get());
const ColumnConstString * col_charset_to = typeid_cast<const ColumnConstString *>(arg_charset_to.column.get());
if (!col_charset_from || !col_charset_to)
throw Exception("2nd and 3rd arguments of function " + getName() + " (source charset and destination charset) must be constant strings.",
ErrorCodes::ILLEGAL_COLUMN);
String charset_from = col_charset_from->getData();
String charset_to = col_charset_to->getData();
if (const ColumnString * col_from = typeid_cast<const ColumnString *>(arg_from.column.get()))
{
auto col_to = std::make_shared<ColumnString>();
convert(charset_from, charset_to, col_from->getChars(), col_from->getOffsets(), col_to->getChars(), col_to->getOffsets());
res.column = col_to;
}
else if (const ColumnConstString * col_from = typeid_cast<const ColumnConstString *>(arg_from.column.get()))
{
auto full_column_holder = col_from->cloneResized(1)->convertToFullColumnIfConst();
const ColumnString * col_from_full = static_cast<const ColumnString *>(full_column_holder.get());
auto col_to_full = std::make_shared<ColumnString>();
convert(charset_from, charset_to, col_from_full->getChars(), col_from_full->getOffsets(), col_to_full->getChars(), col_to_full->getOffsets());
res.column = std::make_shared<ColumnConstString>(col_from->size(), (*col_to_full)[0].get<String>(), res.type);
}
else
throw Exception("Illegal column passed as first argument of function " + getName() + " (must be ColumnString).",
ErrorCodes::ILLEGAL_COLUMN);
}
MatrixOpDataRcPtr MatrixOpData::compose(ConstMatrixOpDataRcPtr & B) const
{
if (getOutputBitDepth() != B->getInputBitDepth())
{
std::ostringstream oss;
oss << "Matrix bit-depth missmatch between '";
oss << getID();
oss << "' and '";
oss << B->getID();
oss << "'. ";
throw Exception(oss.str().c_str());
}
// Ensure that both matrices will have the right dimension (ie. 4x4).
if (m_array.getLength() != 4 || B->m_array.getLength() != 4)
{
// Note: By design, only 4x4 matrices are instantiated.
// The CLF 3x3 (and 3x4) matrices are automatically converted
// to 4x4 matrices, and a Matrix Transform only expects 4x4 matrices.
throw Exception("MatrixOpData: array content issue.");
}
Descriptions newDesc = getDescriptions();
newDesc += B->getDescriptions();
MatrixOpDataRcPtr out = std::make_shared<MatrixOpData>(
getInputBitDepth(),
B->getOutputBitDepth());
out->setID(getID() + B->getID());
out->getDescriptions() = newDesc;
// TODO: May want to revisit how the metadata is set.
// By definition, A.compose(B) implies that op A precedes op B
// in the opList. The LUT format coefficients follow matrix math:
// vec2 = A x vec1 where A is 3x3 and vec is 3x1.
// So the composite operation in matrix form is vec2 = B x A x vec1.
// Hence we compute B x A rather than A x B.
MatrixArrayPtr outPtr = B->m_array.inner(this->m_array);
out->getArray() = *outPtr.get();
// Compute matrix B times offsets from A.
Offsets offs;
B->m_array.inner(getOffsets(), offs);
const unsigned long dim = this->m_array.getLength();
// Determine overall scaling of the offsets prior to any catastrophic
// cancellation that may occur during the add.
double val, max_val = 0.;
for (unsigned long i = 0; i<dim; ++i)
{
val = fabs(offs[i]);
max_val = max_val > val ? max_val : val;
val = fabs(B->getOffsets()[i]);
max_val = max_val > val ? max_val : val;
}
// Add offsets from B.
for (unsigned long i = 0; i<dim; ++i)
{
offs[i] += B->getOffsets()[i];
}
out->setOffsets(offs);
// To enable use of strict float comparisons above, we adjust the
// result so that values very near integers become exactly integers.
out->cleanUp(max_val);
return out;
}
