double ColorHistogram::GenericDistance(
const ColorHistogram& rhs,
std::function<float(float,float)> fun) const {
DCHECK(IsSparse() == rhs.IsSparse());
double sum = 0;
if (!IsSparse()) {
for (int i = 0; i < total_bins_; ++i) {
sum += fun(bins_[i], rhs.bins_[i]);
}
} else {
// Sparse processing.
for (const auto& bin : sparse_bins_) {
const auto& rhs_bin_iter = rhs.sparse_bins_.find(bin.first);
sum += fun(bin.second, rhs_bin_iter != rhs.sparse_bins_.end() ?
rhs_bin_iter->second : 0.0f);
}
// Process rhs bins that we might have missed.
for (const auto& rhs_bin : rhs.sparse_bins_) {
const auto& bin_iter = sparse_bins_.find(rhs_bin.first);
if (bin_iter == sparse_bins_.end()) {
sum += fun(0, rhs_bin.second);
}
}
}
return sum;
}
void NDArrayView::SetValue(double value)
{
if (IsSparse())
LogicError("Filling a NDArrayView with a scalar is only allowed for NDArrayView objects with dense storage format");
GetWritableMatrix<double>()->SetValue(value);
}
void ColorHistogram::ComputeMeanAndVariance() {
DCHECK(IsSparse()) << "Implemented for sparse histograms only.";
DCHECK(IsNormalized()) << "Implemented for normalized histograms only.";
mean_.fill(0);
var_.fill(0);
const ColorHistogramIndexLUT& lut = ColorHistogramIndexLUTFactory::Instance().GetLUT(
lum_bins_, color_bins_, color_bins_);
// Iteration uses simplification that all vals sum to one.
for (const auto& bin : sparse_bins_) {
const std::tuple<int, int, int>& idx_3d = lut.Ind2Sub(bin.first);
const float val = bin.second;
mean_[0] += std::get<0>(idx_3d) * val;
mean_[1] += std::get<1>(idx_3d) * val;
mean_[2] += std::get<2>(idx_3d) * val;
var_[0] += std::get<0>(idx_3d) * std::get<0>(idx_3d) * val;
var_[1] += std::get<1>(idx_3d) * std::get<1>(idx_3d) * val;
var_[2] += std::get<2>(idx_3d) * std::get<2>(idx_3d) * val;
}
var_[0] -= mean_[0] * mean_[0];
var_[1] -= mean_[1] * mean_[1];
var_[2] -= mean_[2] * mean_[2];
}
ColorHistogram ColorHistogram::ScaleHistogram(const vector<float>& gain) const {
const ColorHistogramIndexLUT& lut =
ColorHistogramIndexLUTFactory::Instance().GetLUT(
lum_bins_, color_bins_, color_bins_);
ColorHistogram result = EmptyCopy();
if (!IsSparse()) {
for (int i = 0; i < total_bins_; ++i) {
const float value = bins_[i];
if (value) {
const std::tuple<int, int, int>& idx_3d = lut.Ind2Sub(i);
const float bin_lum = std::min(lum_bins_ - 1.f, std::get<0>(idx_3d) * gain[0]);
const float bin_col1 = std::min(color_bins_ - 1.f, std::get<1>(idx_3d) * gain[1]);
const float bin_col2 = std::min(color_bins_ - 1.f, std::get<2>(idx_3d) * gain[2]);
result.AddValueInterpolated(bin_lum, bin_col1, bin_col2, value);
}
}
} else {
for (const auto& bin : sparse_bins_) {
const std::tuple<int, int, int>& idx_3d = lut.Ind2Sub(bin.first);
const float bin_lum = std::min(lum_bins_ - 1.f, std::get<0>(idx_3d) * gain[0]);
const float bin_col1 = std::min(color_bins_ - 1.f, std::get<1>(idx_3d) * gain[1]);
const float bin_col2 = std::min(color_bins_ - 1.f, std::get<2>(idx_3d) * gain[2]);
result.AddValueInterpolated(bin_lum, bin_col1, bin_col2, bin.second);
}
}
DCHECK_LT(fabs(WeightSum() - result.WeightSum()), 1e-3f);
return result;
}
const ElementType* NDArrayView::DataBuffer() const
{
if (AsDataType<ElementType>() != m_dataType)
LogicError("The specified ElementType %s does not match the DataType %s", typeid(ElementType).name(), DataTypeName(m_dataType));
if (IsSparse())
InvalidArgument("DataBuffer/WritableDataBuffer methods can only be called for NDArrayiew objects with dense storage format");
// First make sure that the underlying matrix is on the right device
auto matrix = GetMatrix<ElementType>();
matrix->TransferToDeviceIfNotThere(AsCNTKImplDeviceId(m_device), true);
return matrix->Data();
}
void ColorHistogram::ConvertToSparse() {
if (IsSparse()) {
DLOG(WARNING) << "Conversion to sparse histogram of already sparse histogram "
"requested. Ignored.";
return;
}
// Anticipate 10% load.
sparse_bins_ = HashBins(total_bins_ / 10);
for (int bin_idx = 0; bin_idx < total_bins_; ++bin_idx) {
const float value = bins_[bin_idx];
if (value != 0) {
sparse_bins_[bin_idx] = value;
}
}
// Free memory.
bins_ = vector<float>();
is_sparse_ = true;
}
/*virtual*/ Dictionary Variable::Serialize() const
{
if (IsOutput())
{
LogicError("Output variables cannot be saved");
}
Dictionary dict;
dict[versionKey] = CurrentVersion();
dict[typeKey] = s_variableTypeValue;
dict[uidKey] = Uid();
dict[kindKey] = static_cast<size_t>(Kind());
dict[dataTypeKey] = static_cast<size_t>(GetDataType());
const auto& dynamicAxes = DynamicAxes();
vector<DictionaryValue> dictionaryValueVector;
dictionaryValueVector.reserve(dynamicAxes.size());
for (const auto& axis : dynamicAxes)
dictionaryValueVector.push_back(axis);
dict[dynamicAxisKey] = dictionaryValueVector;
dict[isSparseKey] = IsSparse();
if (!Name().empty())
dict[nameKey] = Name();
dict[needsGradientKey] = NeedsGradient();
dict[shapeKey] = Shape();
if (IsParameter() || IsConstant())
{
NDArrayView* value = Value().get();
if (value == nullptr)
{
LogicError("Uninitialized Parameter variable cannot be saved");
}
// TODO: add a dictionary value constructor with an rvalue parameter.
dict[valueKey] = DictionaryValue(*value);
}
return dict;
}
void ColorHistogram::NormalizeToOne() {
if (IsNormalized()) {
DLOG(WARNING) << "Normalization of normalized histogram requested. Ignored.";
}
is_normalized_ = true;
if (weight_sum_ == 0) {
return;
}
const float denom = 1.0f / weight_sum_;
if (!IsSparse()) {
for (auto& bin : bins_) {
bin *= denom;
}
} else {
for (auto& bin : sparse_bins_) {
bin.second *= denom;
}
}
}
int GetRealSparseVector(pMatrix ppm, int element, Double *vec, int *index, int start, int len, int col)
{
int j, k, k1, m, n, start1, count = 0;
int *ir, *jc;
double *pr, *pr0;
rMatrix pm = ppm[element];
m = GetM(pm);
n = GetN(pm);
if ( ((col == 0) && (((m != 1) && (n != 1)) || ((m == 1) && (n > len)) || ((n == 1) && (m > len)))) ||
((col != 0) && ((m > len) || (col > n))) ||
!IsNumeric(pm) ||
IsComplex(pm) ) {
ErrMsgTxt("invalid vector.");
}
pr = GetPr(pm);
if (!IsSparse(pm)) {
if ((((n == 1) || (col != 0)) && (m != len)) || ((col == 0) && (m == 1) && (n != len)))
ErrMsgTxt("invalid vector.");
if (col)
pr += (col - 1) * m;
for (k = 0; k < len; k++, pr++) {
if (*pr) {
*(vec++) = *pr;
*(index++) = start + k;
count++;
}
}
} else if (IsSparse(pm)) {
int j1, j2;
jc = mxGetJc(pm);
ir = mxGetIr(pm);
pr0 = pr;
if (col == 0) {
j1 = 0;
j2 = n;
}
else {
j1 = col - 1;
j2 = col;
}
for (j = j1; j < j2; j++) {
k = jc[j];
k1 = jc[j + 1];
pr = pr0 + k;
start1 = start;
if (col == 0)
start1 += j * m;
for (; k < k1; k++, pr++, vec++, index++) {
*vec = *pr;
*index = start1 + ir[k];
count++;
}
}
} else {
ErrMsgTxt("Can't figure out this matrix.");
}
return(count);
}
void ColorHistogram::MergeWithHistogram(const ColorHistogram& rhs) {
DCHECK(is_sparse_ == rhs.is_sparse_) << "Sparsity differs.";
DCHECK(is_normalized_ == rhs.is_normalized_) << "Normalization differs.";
const double n = weight_sum_ + rhs.weight_sum_;
if (n == 0) {
return;
}
// Weighted merge for normalized histograms.
const float n_l = weight_sum_ / n;
const float n_r = rhs.weight_sum_ / n;
// New weight_sum equals sum of both.
weight_sum_ = n;
double weighted_bin_sum = 0;
if (!IsSparse()) {
if (IsNormalized()) {
for (int i = 0; i < total_bins_; ++i) {
bins_[i] = bins_[i] * n_l + rhs.bins_[i] * n_r;
weighted_bin_sum += bins_[i];
}
// Re-Normalize.
const float denom = 1.0f / weighted_bin_sum;
for (float& bin : bins_) {
bin *= denom;
}
} else {
for (int i = 0; i < total_bins_; ++i) {
bins_[i] += rhs.bins_[i];
}
}
} else {
// Sparse version.
if (IsNormalized()) {
for (auto& bin : sparse_bins_) {
const auto rhs_bin_iter = rhs.sparse_bins_.find(bin.first);
if (rhs_bin_iter != rhs.sparse_bins_.end()) {
bin.second = bin.second * n_l + rhs_bin_iter->second * n_r;
} else {
bin.second *= n_l;
}
weighted_bin_sum += bin.second;
}
// Process rhs bins that we might have missed.
for (const auto& rhs_bin : rhs.sparse_bins_) {
const auto bin_iter = sparse_bins_.find(rhs_bin.first);
if (bin_iter == sparse_bins_.end()) {
weighted_bin_sum += (
(sparse_bins_[rhs_bin.first] = rhs_bin.second * n_r));
}
}
// Normalize.
const float denom = 1.0f / weighted_bin_sum;
for (auto& bin : sparse_bins_) {
bin.second *= denom;
}
} else {
for (auto& bin : sparse_bins_) {
const auto rhs_bin_iter = rhs.sparse_bins_.find(bin.first);
if (rhs_bin_iter != rhs.sparse_bins_.end()) {
bin.second += rhs_bin_iter->second;
}
}
// Process rhs bins that we might have missed.
for (const auto& rhs_bin : rhs.sparse_bins_) {
const auto bin_iter = sparse_bins_.find(rhs_bin.first);
if (bin_iter == sparse_bins_.end()) {
sparse_bins_.insert(rhs_bin);
}
}
}
}
}
