void process_sse2(dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, const void *const ivoid,
void *const ovoid, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out)
{
const dt_iop_rawprepare_data_t *const d = (dt_iop_rawprepare_data_t *)piece->data;
// fprintf(stderr, "roi in %d %d %d %d\n", roi_in->x, roi_in->y, roi_in->width, roi_in->height);
// fprintf(stderr, "roi out %d %d %d %d\n", roi_out->x, roi_out->y, roi_out->width, roi_out->height);
const float scale = roi_in->scale / piece->iscale;
const int csx = (int)roundf((float)d->x * scale), csy = (int)roundf((float)d->y * scale);
if(!dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) && piece->pipe->filters)
{ // raw mosaic
#ifdef _OPENMP
#pragma omp parallel for default(none) schedule(static)
#endif
for(int j = 0; j < roi_out->height; j++)
{
const uint16_t *in = ((uint16_t *)ivoid) + ((size_t)roi_in->width * (j + csy) + csx);
float *out = ((float *)ovoid) + (size_t)roi_out->width * j;
int i = 0;
// FIXME: figure alignment! !!! replace with for !!!
while((!dt_is_aligned(in, 16) || !dt_is_aligned(out, 16)) && (i < roi_out->width))
{
const int id = BL(roi_out, d, j, i);
*out = (((float)(*in)) - d->sub[id]) / d->div[id];
i++;
in++;
out++;
}
const __m128 sub = _mm_set_ps(d->sub[BL(roi_out, d, j, i + 3)], d->sub[BL(roi_out, d, j, i + 2)],
d->sub[BL(roi_out, d, j, i + 1)], d->sub[BL(roi_out, d, j, i)]);
const __m128 div = _mm_set_ps(d->div[BL(roi_out, d, j, i + 3)], d->div[BL(roi_out, d, j, i + 2)],
d->div[BL(roi_out, d, j, i + 1)], d->div[BL(roi_out, d, j, i)]);
// process aligned pixels with SSE
for(; i < roi_out->width - (8 - 1); i += 8, in += 8)
{
const __m128i input = _mm_load_si128((__m128i *)in);
__m128i ilo = _mm_unpacklo_epi16(input, _mm_set1_epi16(0));
__m128i ihi = _mm_unpackhi_epi16(input, _mm_set1_epi16(0));
__m128 flo = _mm_cvtepi32_ps(ilo);
__m128 fhi = _mm_cvtepi32_ps(ihi);
flo = _mm_div_ps(_mm_sub_ps(flo, sub), div);
fhi = _mm_div_ps(_mm_sub_ps(fhi, sub), div);
_mm_stream_ps(out, flo);
out += 4;
_mm_stream_ps(out, fhi);
out += 4;
}
// process the rest
for(; i < roi_out->width; i++, in++, out++)
{
const int id = BL(roi_out, d, j, i);
*out = MAX(0.0f, ((float)(*in)) - d->sub[id]) / d->div[id];
}
}
piece->pipe->filters = dt_rawspeed_crop_dcraw_filters(piece->pipe->filters, csx, csy);
adjust_xtrans_filters(piece->pipe->xtrans, csx, csy);
}
else
{ // pre-downsampled buffer that needs black/white scaling
const __m128 sub = _mm_load_ps(d->sub), div = _mm_load_ps(d->div);
#ifdef _OPENMP
#pragma omp parallel for default(none) schedule(static)
#endif
for(int j = 0; j < roi_out->height; j++)
{
const float *in = ((float *)ivoid) + (size_t)4 * (roi_in->width * (j + csy) + csx);
float *out = ((float *)ovoid) + (size_t)4 * roi_out->width * j;
// process aligned pixels with SSE
for(int i = 0; i < roi_out->width; i++, in += 4, out += 4)
{
const __m128 input = _mm_load_ps(in);
const __m128 scaled = _mm_div_ps(_mm_sub_ps(input, sub), div);
_mm_stream_ps(out, scaled);
}
}
}
_mm_sfence();
}
void process(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid,
const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out)
{
const dt_iop_colorout_data_t *const d = (dt_iop_colorout_data_t *)piece->data;
const int ch = piece->colors;
const int gamutcheck = (d->softproof_enabled == DT_SOFTPROOF_GAMUTCHECK);
if(!isnan(d->cmatrix[0]))
{
// fprintf(stderr,"Using cmatrix codepath\n");
// convert to rgb using matrix
#ifdef _OPENMP
#pragma omp parallel for schedule(static) default(none) shared(roi_in, roi_out, ivoid, ovoid)
#endif
for(int j = 0; j < roi_out->height; j++)
{
float *in = (float *)ivoid + (size_t)ch * roi_in->width * j;
float *out = (float *)ovoid + (size_t)ch * roi_out->width * j;
const __m128 m0 = _mm_set_ps(0.0f, d->cmatrix[6], d->cmatrix[3], d->cmatrix[0]);
const __m128 m1 = _mm_set_ps(0.0f, d->cmatrix[7], d->cmatrix[4], d->cmatrix[1]);
const __m128 m2 = _mm_set_ps(0.0f, d->cmatrix[8], d->cmatrix[5], d->cmatrix[2]);
for(int i = 0; i < roi_out->width; i++, in += ch, out += ch)
{
const __m128 xyz = dt_Lab_to_XYZ_SSE(_mm_load_ps(in));
const __m128 t
= _mm_add_ps(_mm_mul_ps(m0, _mm_shuffle_ps(xyz, xyz, _MM_SHUFFLE(0, 0, 0, 0))),
_mm_add_ps(_mm_mul_ps(m1, _mm_shuffle_ps(xyz, xyz, _MM_SHUFFLE(1, 1, 1, 1))),
_mm_mul_ps(m2, _mm_shuffle_ps(xyz, xyz, _MM_SHUFFLE(2, 2, 2, 2)))));
_mm_stream_ps(out, t);
}
}
_mm_sfence();
// apply profile
#ifdef _OPENMP
#pragma omp parallel for schedule(static) default(none) shared(roi_in, roi_out, ivoid, ovoid)
#endif
for(int j = 0; j < roi_out->height; j++)
{
float *in = (float *)ivoid + (size_t)ch * roi_in->width * j;
float *out = (float *)ovoid + (size_t)ch * roi_out->width * j;
for(int i = 0; i < roi_out->width; i++, in += ch, out += ch)
{
for(int i = 0; i < 3; i++)
if(d->lut[i][0] >= 0.0f)
{
out[i] = (out[i] < 1.0f) ? lerp_lut(d->lut[i], out[i])
: dt_iop_eval_exp(d->unbounded_coeffs[i], out[i]);
}
}
}
}
else
{
// fprintf(stderr,"Using xform codepath\n");
const __m128 outofgamutpixel = _mm_set_ps(0.0f, 1.0f, 1.0f, 0.0f);
#ifdef _OPENMP
#pragma omp parallel for schedule(static) default(none) shared(ivoid, ovoid, roi_out)
#endif
for(int k = 0; k < roi_out->height; k++)
{
const float *in = ((float *)ivoid) + (size_t)ch * k * roi_out->width;
float *out = ((float *)ovoid) + (size_t)ch * k * roi_out->width;
if(!gamutcheck)
{
cmsDoTransform(d->xform, in, out, roi_out->width);
}
else
{
void *rgb = dt_alloc_align(16, 4 * sizeof(float) * roi_out->width);
cmsDoTransform(d->xform, in, rgb, roi_out->width);
float *rgbptr = (float *)rgb;
for(int j = 0; j < roi_out->width; j++, rgbptr += 4, out += 4)
{
const __m128 pixel = _mm_load_ps(rgbptr);
__m128 ingamut = _mm_cmplt_ps(pixel, _mm_set_ps(-FLT_MAX, 0.0f, 0.0f, 0.0f));
ingamut = _mm_or_ps(_mm_unpacklo_ps(ingamut, ingamut), _mm_unpackhi_ps(ingamut, ingamut));
ingamut = _mm_or_ps(_mm_unpacklo_ps(ingamut, ingamut), _mm_unpackhi_ps(ingamut, ingamut));
const __m128 result
= _mm_or_ps(_mm_and_ps(ingamut, outofgamutpixel), _mm_andnot_ps(ingamut, pixel));
_mm_stream_ps(out, result);
}
dt_free_align(rgb);
}
}
_mm_sfence();
}
if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height);
}
int main()
{
float *arr = get_arr(); // [4, 3, 2, 1]
float *uarr = get_uarr(); // [5, 4, 3, 2]
float *arr2 = get_arr2(); // [4, 3, 2, 1]
float *uarr2 = get_uarr2(); // [5, 4, 3, 2]
__m128 a = get_a(); // [8, 6, 4, 2]
__m128 b = get_b(); // [1, 2, 3, 4]
// Check that test data is like expected.
Assert(((uintptr_t)arr & 0xF) == 0); // arr must be aligned by 16.
Assert(((uintptr_t)uarr & 0xF) != 0); // uarr must be unaligned.
Assert(((uintptr_t)arr2 & 0xF) == 0); // arr must be aligned by 16.
Assert(((uintptr_t)uarr2 & 0xF) != 0); // uarr must be unaligned.
// Test that aeq itself works and does not trivially return true on everything.
Assert(aeq_("",_mm_load_ps(arr), 4.f, 3.f, 2.f, 0.f, false) == false);
#ifdef TEST_M64
Assert(aeq64(u64castm64(0x22446688AACCEEFFULL), 0xABABABABABABABABULL, false) == false);
#endif
// SSE1 Load instructions:
aeq(_mm_load_ps(arr), 4.f, 3.f, 2.f, 1.f); // 4-wide load from aligned address.
aeq(_mm_load_ps1(uarr), 2.f, 2.f, 2.f, 2.f); // Load scalar from unaligned address and populate 4-wide.
aeq(_mm_load_ss(uarr), 0.f, 0.f, 0.f, 2.f); // Load scalar from unaligned address to lowest, and zero all highest.
aeq(_mm_load1_ps(uarr), 2.f, 2.f, 2.f, 2.f); // _mm_load1_ps == _mm_load_ps1
aeq(_mm_loadh_pi(a, (__m64*)uarr), 3.f, 2.f, 4.f, 2.f); // Load two highest addresses, preserve two lowest.
aeq(_mm_loadl_pi(a, (__m64*)uarr), 8.f, 6.f, 3.f, 2.f); // Load two lowest addresses, preserve two highest.
aeq(_mm_loadr_ps(arr), 1.f, 2.f, 3.f, 4.f); // 4-wide load from an aligned address, but reverse order.
aeq(_mm_loadu_ps(uarr), 5.f, 4.f, 3.f, 2.f); // 4-wide load from an unaligned address.
// SSE1 Set instructions:
aeq(_mm_set_ps(uarr[3], 2.f, 3.f, 4.f), 5.f, 2.f, 3.f, 4.f); // 4-wide set by specifying four immediate or memory operands.
aeq(_mm_set_ps1(uarr[3]), 5.f, 5.f, 5.f, 5.f); // 4-wide set by specifying one scalar that is expanded.
aeq(_mm_set_ss(uarr[3]), 0.f, 0.f, 0.f, 5.f); // Set scalar at lowest index, zero all higher.
aeq(_mm_set1_ps(uarr[3]), 5.f, 5.f, 5.f, 5.f); // _mm_set1_ps == _mm_set_ps1
aeq(_mm_setr_ps(uarr[3], 2.f, 3.f, 4.f), 4.f, 3.f, 2.f, 5.f); // 4-wide set by specifying four immediate or memory operands, but reverse order.
aeq(_mm_setzero_ps(), 0.f, 0.f, 0.f, 0.f); // Returns a new zero register.
// SSE1 Move instructions:
aeq(_mm_move_ss(a, b), 8.f, 6.f, 4.f, 4.f); // Copy three highest elements from a, and lowest from b.
aeq(_mm_movehl_ps(a, b), 8.f, 6.f, 1.f, 2.f); // Copy two highest elements from a, and take two highest from b and place them to the two lowest in output.
aeq(_mm_movelh_ps(a, b), 3.f, 4.f, 4.f, 2.f); // Copy two lowest elements from a, and take two lowest from b and place them to the two highest in output.
// SSE1 Store instructions:
#ifdef TEST_M64
/*M64*/*(uint64_t*)uarr = 0xCDCDCDCDCDCDCDCDULL; _mm_maskmove_si64(u64castm64(0x00EEDDCCBBAA9988ULL), u64castm64(0x0080FF7F01FEFF40ULL), (char*)uarr); Assert(*(uint64_t*)uarr == 0xCDEEDDCDCDAA99CDULL); // _mm_maskmove_si64: Conditionally store bytes of a 64-bit value.
/*M64*/*(uint64_t*)uarr = 0xABABABABABABABABULL; _m_maskmovq(u64castm64(0x00EEDDCCBBAA9988ULL), u64castm64(0x0080FF7F01FEFF40ULL), (char*)uarr); Assert(*(uint64_t*)uarr == 0xABEEDDABABAA99ABULL); // _m_maskmovq is an alias to _mm_maskmove_si64.
#endif
_mm_store_ps(arr2, a); aeq(_mm_load_ps(arr2), 8.f, 6.f, 4.f, 2.f); // _mm_store_ps: 4-wide store to aligned memory address.
_mm_store_ps1(arr2, a); aeq(_mm_load_ps(arr2), 2.f, 2.f, 2.f, 2.f); // _mm_store_ps1: Store lowest scalar to aligned address, duplicating the element 4 times.
_mm_storeu_ps(uarr2, _mm_set1_ps(100.f)); _mm_store_ss(uarr2, b); aeq(_mm_loadu_ps(uarr2), 100.f, 100.f, 100.f, 4.f); // _mm_store_ss: Store lowest scalar to unaligned address. Don't adjust higher addresses in memory.
_mm_store_ps(arr2, _mm_set1_ps(100.f)); _mm_store1_ps(arr2, a); aeq(_mm_load_ps(arr2), 2.f, 2.f, 2.f, 2.f); // _mm_store1_ps == _mm_store_ps1
_mm_storeu_ps(uarr2, _mm_set1_ps(100.f)); _mm_storeh_pi((__m64*)uarr2, a); aeq(_mm_loadu_ps(uarr2), 100.f, 100.f, 8.f, 6.f); // _mm_storeh_pi: Store two highest elements to memory.
_mm_storeu_ps(uarr2, _mm_set1_ps(100.f)); _mm_storel_pi((__m64*)uarr2, a); aeq(_mm_loadu_ps(uarr2), 100.f, 100.f, 4.f, 2.f); // _mm_storel_pi: Store two lowest elements to memory.
_mm_storer_ps(arr2, a); aeq(_mm_load_ps(arr2), 2.f, 4.f, 6.f, 8.f); // _mm_storer_ps: 4-wide store to aligned memory address, but reverse the elements on output.
_mm_storeu_ps(uarr2, a); aeq(_mm_loadu_ps(uarr2), 8.f, 6.f, 4.f, 2.f); // _mm_storeu_ps: 4-wide store to unaligned memory address.
#ifdef TEST_M64
/*M64*/_mm_stream_pi((__m64*)uarr, u64castm64(0x0080FF7F01FEFF40ULL)); Assert(*(uint64_t*)uarr == 0x0080FF7F01FEFF40ULL); // _mm_stream_pi: 2-wide store, but with a non-temporal memory cache hint.
#endif
_mm_store_ps(arr2, _mm_set1_ps(100.f)); _mm_stream_ps(arr2, a); aeq(_mm_load_ps(arr2), 8.f, 6.f, 4.f, 2.f); // _mm_stream_ps: 4-wide store, but with a non-temporal memory cache hint.
// SSE1 Arithmetic instructions:
aeq(_mm_add_ps(a, b), 9.f, 8.f, 7.f, 6.f); // 4-wide add.
aeq(_mm_add_ss(a, b), 8.f, 6.f, 4.f, 6.f); // Add lowest element, preserve three highest unchanged from a.
aeq(_mm_div_ps(a, _mm_set_ps(2.f, 3.f, 8.f, 2.f)), 4.f, 2.f, 0.5f, 1.f); // 4-wide div.
aeq(_mm_div_ss(a, _mm_set_ps(2.f, 3.f, 8.f, 8.f)), 8.f, 6.f, 4.f, 0.25f); // Div lowest element, preserve three highest unchanged from a.
aeq(_mm_mul_ps(a, b), 8.f, 12.f, 12.f, 8.f); // 4-wide mul.
aeq(_mm_mul_ss(a, b), 8.f, 6.f, 4.f, 8.f); // Mul lowest element, preserve three highest unchanged from a.
#ifdef TEST_M64
__m64 m1 = get_m1();
/*M64*/aeq64(_mm_mulhi_pu16(m1, u64castm64(0x22446688AACCEEFFULL)), 0x002233440B4C33CFULL); // Multiply u16 channels, and store high parts.
/*M64*/aeq64( _m_pmulhuw(m1, u64castm64(0x22446688AACCEEFFULL)), 0x002233440B4C33CFULL); // _m_pmulhuw is an alias to _mm_mulhi_pu16.
__m64 m2 = get_m2();
/*M64*/aeq64(_mm_sad_pu8(m1, m2), 0x368ULL); // Compute abs. differences of u8 channels, and sum those up to a single 16-bit scalar.
/*M64*/aeq64( _m_psadbw(m1, m2), 0x368ULL); // _m_psadbw is an alias to _mm_sad_pu8.
#endif
aeq(_mm_sub_ps(a, b), 7.f, 4.f, 1.f, -2.f); // 4-wide sub.
aeq(_mm_sub_ss(a, b), 8.f, 6.f, 4.f, -2.f); // Sub lowest element, preserve three highest unchanged from a.
// SSE1 Elementary Math functions:
#ifndef __EMSCRIPTEN__ // TODO: Enable support for this to pass.
aeq(_mm_rcp_ps(a), 0.124969f, 0.166626f, 0.249939f, 0.499878f); // Compute 4-wide 1/x.
aeq(_mm_rcp_ss(a), 8.f, 6.f, 4.f, 0.499878f); // Compute 1/x of lowest element, pass higher elements unchanged.
aeq(_mm_rsqrt_ps(a), 0.353455f, 0.408203f, 0.499878f, 0.706909f); // Compute 4-wide 1/sqrt(x).
aeq(_mm_rsqrt_ss(a), 8.f, 6.f, 4.f, 0.706909f); // Compute 1/sqrt(x) of lowest element, pass higher elements unchanged.
#endif
aeq(_mm_sqrt_ps(a), 2.82843f, 2.44949f, 2.f, 1.41421f); // Compute 4-wide sqrt(x).
aeq(_mm_sqrt_ss(a), 8.f, 6.f, 4.f, 1.41421f); // Compute sqrt(x) of lowest element, pass higher elements unchanged.
__m128 i1 = get_i1();
__m128 i2 = get_i2();
// SSE1 Logical instructions:
#ifndef __EMSCRIPTEN__ // TODO: The polyfill currently does NaN canonicalization and breaks these.
aeqi(_mm_and_ps(i1, i2), 0x83200100, 0x0fecc988, 0x80244021, 0x13458a88); // 4-wide binary AND
aeqi(_mm_andnot_ps(i1, i2), 0x388a9888, 0xf0021444, 0x7000289c, 0x00121046); // 4-wide binary (!i1) & i2
aeqi(_mm_or_ps(i1, i2), 0xbfefdba9, 0xffefdfed, 0xf7656bbd, 0xffffdbef); // 4-wide binary OR
aeqi(_mm_xor_ps(i1, i2), 0x3ccfdaa9, 0xf0031665, 0x77412b9c, 0xecba5167); // 4-wide binary XOR
#endif
// SSE1 Compare instructions:
// a = [8, 6, 4, 2], b = [1, 2, 3, 4]
aeqi(_mm_cmpeq_ps(a, _mm_set_ps(8.f, 0.f, 4.f, 0.f)), 0xFFFFFFFF, 0, 0xFFFFFFFF, 0); // 4-wide cmp ==
aeqi(_mm_cmpeq_ss(a, _mm_set_ps(8.f, 0.f, 4.f, 2.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0xFFFFFFFF); // scalar cmp ==, pass three highest unchanged.
aeqi(_mm_cmpge_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0xFFFFFFFF, 0, 0xFFFFFFFF, 0); // 4-wide cmp >=
aeqi(_mm_cmpge_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 0.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0xFFFFFFFF); // scalar cmp >=, pass three highest unchanged.
aeqi(_mm_cmpgt_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0, 0, 0xFFFFFFFF, 0); // 4-wide cmp >
aeqi(_mm_cmpgt_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 2.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0); // scalar cmp >, pass three highest unchanged.
aeqi(_mm_cmple_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0xFFFFFFFF, 0xFFFFFFFF, 0, 0xFFFFFFFF); // 4-wide cmp <=
aeqi(_mm_cmple_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 0.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0); // scalar cmp <=, pass three highest unchanged.
aeqi(_mm_cmplt_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0, 0xFFFFFFFF, 0, 0xFFFFFFFF); // 4-wide cmp <
aeqi(_mm_cmplt_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 2.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0); // scalar cmp <, pass three highest unchanged.
aeqi(_mm_cmpneq_ps(a, _mm_set_ps(8.f, 0.f, 4.f, 0.f)), 0, 0xFFFFFFFF, 0, 0xFFFFFFFF); // 4-wide cmp !=
aeqi(_mm_cmpneq_ss(a, _mm_set_ps(8.f, 0.f, 4.f, 2.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0); // scalar cmp !=, pass three highest unchanged.
aeqi(_mm_cmpnge_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0, 0xFFFFFFFF, 0, 0xFFFFFFFF); // 4-wide cmp not >=
aeqi(_mm_cmpnge_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 0.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0); // scalar cmp not >=, pass three highest unchanged.
aeqi(_mm_cmpngt_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0xFFFFFFFF, 0xFFFFFFFF, 0, 0xFFFFFFFF); // 4-wide cmp not >
aeqi(_mm_cmpngt_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 2.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0xFFFFFFFF); // scalar cmp not >, pass three highest unchanged.
aeqi(_mm_cmpnle_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0, 0, 0xFFFFFFFF, 0); // 4-wide cmp not <=
aeqi(_mm_cmpnle_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 0.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0xFFFFFFFF); // scalar cmp not <=, pass three highest unchanged.
aeqi(_mm_cmpnlt_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0xFFFFFFFF, 0, 0xFFFFFFFF, 0); // 4-wide cmp not <
aeqi(_mm_cmpnlt_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 2.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0xFFFFFFFF); // scalar cmp not <, pass three highest unchanged.
__m128 nan1 = get_nan1(); // [NAN, 0, 0, NAN]
__m128 nan2 = get_nan2(); // [NAN, NAN, 0, 0]
aeqi(_mm_cmpord_ps(nan1, nan2), 0, 0, 0xFFFFFFFF, 0); // 4-wide test if both operands are not nan.
aeqi(_mm_cmpord_ss(nan1, nan2), fcastu(NAN), 0, 0, 0); // scalar test if both operands are not nan, pass three highest unchanged.
// Intel Intrinsics Guide documentation is wrong on _mm_cmpunord_ps and _mm_cmpunord_ss. MSDN is right: http://msdn.microsoft.com/en-us/library/khy6fk1t(v=vs.90).aspx
aeqi(_mm_cmpunord_ps(nan1, nan2), 0xFFFFFFFF, 0xFFFFFFFF, 0, 0xFFFFFFFF); // 4-wide test if one of the operands is nan.
#ifndef __EMSCRIPTEN__ // TODO: The polyfill currently does NaN canonicalization and breaks these.
aeqi(_mm_cmpunord_ss(nan1, nan2), fcastu(NAN), 0, 0, 0xFFFFFFFF); // scalar test if one of the operands is nan, pass three highest unchanged.
#endif
Assert(_mm_comieq_ss(a, b) == 0); Assert(_mm_comieq_ss(a, a) == 1); // Scalar cmp == of lowest element, return int.
Assert(_mm_comige_ss(a, b) == 0); Assert(_mm_comige_ss(a, a) == 1); // Scalar cmp >= of lowest element, return int.
Assert(_mm_comigt_ss(b, a) == 1); Assert(_mm_comigt_ss(a, a) == 0); // Scalar cmp > of lowest element, return int.
Assert(_mm_comile_ss(b, a) == 0); Assert(_mm_comile_ss(a, a) == 1); // Scalar cmp <= of lowest element, return int.
Assert(_mm_comilt_ss(a, b) == 1); Assert(_mm_comilt_ss(a, a) == 0); // Scalar cmp < of lowest element, return int.
Assert(_mm_comineq_ss(a, b) == 1); Assert(_mm_comineq_ss(a, a) == 0); // Scalar cmp != of lowest element, return int.
// The ucomi versions are identical to comi, except that ucomi signal a FP exception only if one of the input operands is a SNaN, whereas the comi versions signal a FP
// exception when one of the input operands is either a QNaN or a SNaN.
#ifndef __EMSCRIPTEN__ // TODO: Fix ucomi support in SSE to treat NaNs properly.
Assert(_mm_ucomieq_ss(a, b) == 0); Assert(_mm_ucomieq_ss(a, a) == 1); Assert(_mm_ucomieq_ss(a, nan1) == 1);
#endif
Assert(_mm_ucomige_ss(a, b) == 0); Assert(_mm_ucomige_ss(a, a) == 1); Assert(_mm_ucomige_ss(a, nan1) == 0);
Assert(_mm_ucomigt_ss(b, a) == 1); Assert(_mm_ucomigt_ss(a, a) == 0); Assert(_mm_ucomigt_ss(a, nan1) == 0);
Assert(_mm_ucomile_ss(b, a) == 0); Assert(_mm_ucomile_ss(a, a) == 1); Assert(_mm_ucomile_ss(a, nan1) == 1);
Assert(_mm_ucomilt_ss(a, b) == 1); Assert(_mm_ucomilt_ss(a, a) == 0); Assert(_mm_ucomilt_ss(a, nan1) == 1);
#ifndef __EMSCRIPTEN__ // TODO: Fix ucomi support in SSE to treat NaNs properly.
Assert(_mm_ucomineq_ss(a, b) == 1); Assert(_mm_ucomineq_ss(a, a) == 0); Assert(_mm_ucomineq_ss(a, nan1) == 0);
#endif
// SSE1 Convert instructions:
__m128 c = get_c(); // [1.5, 2.5, 3.5, 4.5]
__m128 e = get_e(); // [INF, -INF, 2.5, 3.5]
__m128 f = get_f(); // [-1.5, 1.5, -2.5, -9223372036854775808]
#ifdef TEST_M64
/*M64*/aeq(_mm_cvt_pi2ps(a, m2), 8.f, 6.f, -19088744.f, 1985229312.f); // 2-way int32 to float conversion to two lowest channels of m128.
/*M64*/aeq64(_mm_cvt_ps2pi(c), 0x400000004ULL); // 2-way two lowest floats from m128 to integer, return as m64.
#endif
aeq(_mm_cvtsi32_ss(c, -16777215), 1.5f, 2.5f, 3.5f, -16777215.f); // Convert int to float, store in lowest channel of m128.
aeq( _mm_cvt_si2ss(c, -16777215), 1.5f, 2.5f, 3.5f, -16777215.f); // _mm_cvt_si2ss is an alias to _mm_cvtsi32_ss.
#ifndef __EMSCRIPTEN__ // TODO: Fix banker's rounding in cvt functions.
Assert(_mm_cvtss_si32(c) == 4); Assert(_mm_cvtss_si32(e) == 4); // Convert lowest channel of m128 from float to int.
Assert( _mm_cvt_ss2si(c) == 4); Assert( _mm_cvt_ss2si(e) == 4); // _mm_cvt_ss2si is an alias to _mm_cvtss_si32.
#endif
#ifdef TEST_M64
/*M64*/aeq(_mm_cvtpi16_ps(m1), 255.f , -32767.f, 4336.f, 14207.f); // 4-way convert int16s to floats, return in a m128.
/*M64*/aeq(_mm_cvtpi32_ps(a, m1), 8.f, 6.f, 16744449.f, 284178304.f); // 2-way convert int32s to floats, return in two lowest channels of m128, pass two highest unchanged.
/*M64*/aeq(_mm_cvtpi32x2_ps(m1, m2), -19088744.f, 1985229312.f, 16744449.f, 284178304.f); // 4-way convert int32s from two different m64s to float.
/*M64*/aeq(_mm_cvtpi8_ps(m1), 16.f, -16.f, 55.f, 127.f); // 4-way convert int8s from lowest end of m64 to float in a m128.
/*M64*/aeq64(_mm_cvtps_pi16(c), 0x0002000200040004ULL); // 4-way convert floats to int16s in a m64.
/*M64*/aeq64(_mm_cvtps_pi32(c), 0x0000000400000004ULL); // 2-way convert two lowest floats to int32s in a m64.
/*M64*/aeq64(_mm_cvtps_pi8(c), 0x0000000002020404ULL); // 4-way convert floats to int8s in a m64, zero higher half of the returned m64.
/*M64*/aeq(_mm_cvtpu16_ps(m1), 255.f , 32769.f, 4336.f, 14207.f); // 4-way convert uint16s to floats, return in a m128.
/*M64*/aeq(_mm_cvtpu8_ps(m1), 16.f, 240.f, 55.f, 127.f); // 4-way convert uint8s from lowest end of m64 to float in a m128.
#endif
aeq(_mm_cvtsi64_ss(c, -9223372036854775808ULL), 1.5f, 2.5f, 3.5f, -9223372036854775808.f); // Convert single int64 to float, store in lowest channel of m128, and pass three higher channel unchanged.
Assert(_mm_cvtss_f32(c) == 4.5f); // Extract lowest channel of m128 to a plain old float.
Assert(_mm_cvtss_si64(f) == -9223372036854775808ULL); // Convert lowest channel of m128 from float to int64.
#ifdef TEST_M64
/*M64*/aeq64(_mm_cvtt_ps2pi(e), 0x0000000200000003ULL); aeq64(_mm_cvtt_ps2pi(f), 0xfffffffe80000000ULL); // Truncating conversion from two lowest floats of m128 to int32s, return in a m64.
#endif
Assert(_mm_cvttss_si32(e) == 3); // Truncating conversion from the lowest float of a m128 to int32.
Assert( _mm_cvtt_ss2si(e) == 3); // _mm_cvtt_ss2si is an alias to _mm_cvttss_si32.
#ifdef TEST_M64
/*M64*/aeq64(_mm_cvttps_pi32(c), 0x0000000300000004ULL); // Truncating conversion from two lowest floats of m128 to m64.
#endif
Assert(_mm_cvttss_si64(f) == -9223372036854775808ULL); // Truncating conversion from lowest channel of m128 from float to int64.
#ifndef __EMSCRIPTEN__ // TODO: Not implemented.
// SSE1 General support:
unsigned int mask = _MM_GET_EXCEPTION_MASK();
_MM_SET_EXCEPTION_MASK(mask);
unsigned int flushZeroMode = _MM_GET_FLUSH_ZERO_MODE();
_MM_SET_FLUSH_ZERO_MODE(flushZeroMode);
unsigned int roundingMode = _MM_GET_ROUNDING_MODE();
_MM_SET_ROUNDING_MODE(roundingMode);
unsigned int csr = _mm_getcsr();
_mm_setcsr(csr);
unsigned char dummyData[4096];
_mm_prefetch(dummyData, _MM_HINT_T0);
_mm_prefetch(dummyData, _MM_HINT_T1);
_mm_prefetch(dummyData, _MM_HINT_T2);
_mm_prefetch(dummyData, _MM_HINT_NTA);
_mm_sfence();
#endif
// SSE1 Misc instructions:
#ifdef TEST_M64
/*M64*/Assert(_mm_movemask_pi8(m1) == 100); // Return int with eight lowest bits set depending on the highest bits of the 8 uint8 input channels of the m64.
/*M64*/Assert( _m_pmovmskb(m1) == 100); // _m_pmovmskb is an alias to _mm_movemask_pi8.
#endif
Assert(_mm_movemask_ps(_mm_set_ps(-1.f, 0.f, 1.f, NAN)) == 8); Assert(_mm_movemask_ps(_mm_set_ps(-INFINITY, -0.f, INFINITY, -INFINITY)) == 13); // Return int with four lowest bits set depending on the highest bits of the 4 m128 input channels.
// SSE1 Probability/Statistics instructions:
#ifdef TEST_M64
/*M64*/aeq64(_mm_avg_pu16(m1, m2), 0x7FEE9D4D43A234C8ULL); // 4-way average uint16s.
/*M64*/aeq64( _m_pavgw(m1, m2), 0x7FEE9D4D43A234C8ULL); // _m_pavgw is an alias to _mm_avg_pu16.
/*M64*/aeq64(_mm_avg_pu8(m1, m2), 0x7FEE9D4D43A23548ULL); // 8-way average uint8s.
/*M64*/aeq64( _m_pavgb(m1, m2), 0x7FEE9D4D43A23548ULL); // _m_pavgb is an alias to _mm_avg_pu8.
// SSE1 Special Math instructions:
/*M64*/aeq64(_mm_max_pi16(m1, m2), 0xFFBA987654377FULL); // 4-way average uint16s.
/*M64*/aeq64( _m_pmaxsw(m1, m2), 0xFFBA987654377FULL); // _m_pmaxsw is an alias to _mm_max_pi16.
/*M64*/aeq64(_mm_max_pu8(m1, m2), 0xFEFFBA9876F0377FULL); // 4-way average uint16s.
/*M64*/aeq64( _m_pmaxub(m1, m2), 0xFEFFBA9876F0377FULL); // _m_pmaxub is an alias to _mm_max_pu8.
/*M64*/aeq64(_mm_min_pi16(m1, m2), 0xFEDC800110F03210ULL); // 4-way average uint16s.
/*M64*/aeq64( _m_pminsw(m1, m2), 0xFEDC800110F03210ULL); // is an alias to _mm_min_pi16.
/*M64*/aeq64(_mm_min_pu8(m1, m2), 0xDC800110543210ULL); // 4-way average uint16s.
/*M64*/aeq64( _m_pminub(m1, m2), 0xDC800110543210ULL); // is an alias to _mm_min_pu8.
#endif
// a = [8, 6, 4, 2], b = [1, 2, 3, 4]
aeq(_mm_max_ps(a, b), 8.f, 6.f, 4.f, 4.f); // 4-wide max.
aeq(_mm_max_ss(a, _mm_set1_ps(100.f)), 8.f, 6.f, 4.f, 100.f); // Scalar max, pass three highest unchanged.
aeq(_mm_min_ps(a, b), 1.f, 2.f, 3.f, 2.f); // 4-wide min.
aeq(_mm_min_ss(a, _mm_set1_ps(-100.f)), 8.f, 6.f, 4.f, -100.f); // Scalar min, pass three highest unchanged.
// SSE1 Swizzle instructions:
#ifdef TEST_M64
/*M64*/Assert(_mm_extract_pi16(m1, 1) == 4336); // Extract the given int16 channel from a m64.
/*M64*/Assert( _m_pextrw(m1, 1) == 4336); // _m_pextrw is an alias to _mm_extract_pi16.
/*M64*/aeq64(_mm_insert_pi16(m1, 0xABCD, 1), 0xFF8001ABCD377FULL); // Insert a int16 to a specific channel of a m64.
/*M64*/aeq64( _m_pinsrw(m1, 0xABCD, 1), 0xFF8001ABCD377FULL); // _m_pinsrw is an alias to _mm_insert_pi16.
/*M64*/aeq64(_mm_shuffle_pi16(m1, _MM_SHUFFLE(1, 0, 3, 2)), 0x10F0377F00FF8001ULL); // Shuffle int16s around in the 4 channels of the m64.
/*M64*/aeq64( _m_pshufw(m1, _MM_SHUFFLE(1, 0, 3, 2)), 0x10F0377F00FF8001ULL); // _m_pshufw is an alias to _mm_shuffle_pi16.
#endif
aeq(_mm_shuffle_ps(a, b, _MM_SHUFFLE(1, 0, 3, 2)), 3.f, 4.f, 8.f, 6.f);
aeq(_mm_unpackhi_ps(a, b), 1.f , 8.f, 2.f, 6.f);
aeq(_mm_unpacklo_ps(a, b), 3.f , 4.f, 4.f, 2.f);
// Transposing a matrix via the xmmintrin.h-provided intrinsic.
__m128 c0 = a; // [8, 6, 4, 2]
__m128 c1 = b; // [1, 2, 3, 4]
__m128 c2 = get_c(); // [1.5, 2.5, 3.5, 4.5]
__m128 c3 = get_d(); // [8.5, 6.5, 4.5, 2.5]
_MM_TRANSPOSE4_PS(c0, c1, c2, c3);
aeq(c0, 2.5f, 4.5f, 4.f, 2.f);
aeq(c1, 4.5f, 3.5f, 3.f, 4.f);
aeq(c2, 6.5f, 2.5f, 2.f, 6.f);
aeq(c3, 8.5f, 1.5f, 1.f, 8.f);
// All done!
if (numFailures == 0)
printf("Success!\n");
else
printf("%d tests failed!\n", numFailures);
}
void process(
struct dt_iop_module_t *self,
dt_dev_pixelpipe_iop_t *piece,
void *ivoid,
void *ovoid,
const dt_iop_roi_t *roi_in,
const dt_iop_roi_t *roi_out)
{
const int filters = dt_image_flipped_filter(&piece->pipe->image);
dt_iop_highlights_data_t *data = (dt_iop_highlights_data_t *)piece->data;
const float clip = data->clip * fminf(piece->pipe->processed_maximum[0], fminf(piece->pipe->processed_maximum[1], piece->pipe->processed_maximum[2]));
// const int ch = piece->colors;
if(dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) || !filters)
{
const __m128 clipm = _mm_set1_ps(clip);
#ifdef _OPENMP
#pragma omp parallel for schedule(dynamic) default(none) shared(ovoid, ivoid, roi_in, roi_out, data, piece)
#endif
for(int j=0; j<roi_out->height; j++)
{
float *out = (float *)ovoid + (size_t)4*roi_out->width*j;
float *in = (float *)ivoid + (size_t)4*roi_in->width*j;
for(int i=0; i<roi_out->width; i++)
{
_mm_stream_ps(out, _mm_min_ps(clipm, _mm_set_ps(in[3],in[2],in[1],in[0])));
in += 4;
out += 4;
}
}
_mm_sfence();
return;
}
switch(data->mode)
{
case DT_IOP_HIGHLIGHTS_INPAINT: // a1ex's (magiclantern) idea of color inpainting:
{
const float clips[4] = {
0.987*data->clip * piece->pipe->processed_maximum[0],
0.987*data->clip * piece->pipe->processed_maximum[1],
0.987*data->clip * piece->pipe->processed_maximum[2],
clip};
#ifdef _OPENMP
#pragma omp parallel for schedule(dynamic) default(none) shared(ovoid, ivoid, roi_in, roi_out, data, piece)
#endif
for(int j=0; j<roi_out->height; j++)
{
_interpolate_color(ivoid, ovoid, roi_out, 0, 1, j, clips, filters, 0);
_interpolate_color(ivoid, ovoid, roi_out, 0, -1, j, clips, filters, 1);
}
// up/down directions
#ifdef _OPENMP
#pragma omp parallel for schedule(dynamic) default(none) shared(ovoid, ivoid, roi_in, roi_out, data, piece)
#endif
for(int i=0; i<roi_out->width; i++)
{
_interpolate_color(ivoid, ovoid, roi_out, 1, 1, i, clips, filters, 2);
_interpolate_color(ivoid, ovoid, roi_out, 1, -1, i, clips, filters, 3);
}
break;
}
case DT_IOP_HIGHLIGHTS_LCH:
#ifdef _OPENMP
#pragma omp parallel for schedule(dynamic) default(none) shared(ovoid, ivoid, roi_in, roi_out, data, piece)
#endif
for(int j=0; j<roi_out->height; j++)
{
float *out = (float *)ovoid + (size_t)roi_out->width*j;
float *in = (float *)ivoid + (size_t)roi_out->width*j;
for(int i=0; i<roi_out->width; i++)
{
if(i==0 || i==roi_out->width-1 || j==0 || j==roi_out->height-1)
{
// fast path for border
out[0] = in[0];
}
else
{
// analyse one bayer block to get same number of rggb pixels each time
const float near_clip = 0.96f*clip;
const float post_clip = 1.10f*clip;
float blend = 0.0f;
float mean = 0.0f;
for(int jj=0; jj<=1; jj++)
{
for(int ii=0; ii<=1; ii++)
{
const float val = in[(size_t)jj*roi_out->width + ii];
mean += val*0.25f;
blend += (fminf(post_clip, val) - near_clip)/(post_clip-near_clip);
}
}
blend = CLAMP(blend, 0.0f, 1.0f);
if(blend > 0)
{
// recover:
out[0] = blend*mean + (1.f-blend)*in[0];
}
else out[0] = in[0];
}
out ++;
in ++;
}
}
break;
default:
case DT_IOP_HIGHLIGHTS_CLIP:
{
const __m128 clipm = _mm_set1_ps(clip);
const size_t n = (size_t)roi_out->height*roi_out->width;
float *const out = (float *)ovoid;
float *const in = (float *)ivoid;
#ifdef _OPENMP
#pragma omp parallel for schedule(static) default(none)
#endif
for(int j=0; j<n; j+=4)
_mm_stream_ps(out+j, _mm_min_ps(clipm, _mm_load_ps(in+j)));
_mm_sfence();
// lets see if there's a non-multiple of four rest to process:
if(n & 3) for(size_t j=n&~3u; j<n; j++) out[j] = MIN(clip, in[j]);
break;
}
}
if(piece->pipe->mask_display)
dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height);
}
int sse3_ChirpData_ak8(
sah_complex * cx_DataArray,
sah_complex * cx_ChirpDataArray,
int chirp_rate_ind,
double chirp_rate,
int ul_NumDataPoints,
double sample_rate
) {
#ifdef USE_MANUAL_CALLSTACK
call_stack.enter("sse3_ChirpData_ak8()");
#endif
int i;
if (chirp_rate_ind == 0) {
memcpy(cx_ChirpDataArray, cx_DataArray, (int)ul_NumDataPoints * sizeof(sah_complex) );
#ifdef USE_MANUAL_CALLSTACK
call_stack.exit();
#endif
return 0;
}
int vEnd;
double srate = chirp_rate * 0.5 / (sample_rate * sample_rate);
__m128d rate = _mm_set1_pd(chirp_rate * 0.5 / (sample_rate * sample_rate));
__m128d roundVal = _mm_set1_pd(srate >= 0.0 ? TWO_TO_52 : -TWO_TO_52);
__m128d DFOUR = _mm_set_pd(4.0, 4.0);
// main vectorised loop
vEnd = ul_NumDataPoints - (ul_NumDataPoints & 3);
__m128d di1 = _mm_set_pd(2.0, 0.0); // set time patterns for eventual moveldup/movehdup
__m128d di2 = _mm_set_pd(3.0, 1.0);
for (i = 0; i < vEnd; i += 4) {
const float *d = (const float *) (cx_DataArray + i);
float *cd = (float *) (cx_ChirpDataArray + i);
__m128d a1, a2;
__m128 d1, d2;
__m128 cd1, cd2;
__m128 td1, td2;
__m128 x;
__m128 y;
__m128 z;
__m128 s;
__m128 c;
__m128 m;
// load the signal to be chirped
d1 = _mm_load_ps(d);
d2 = _mm_load_ps(d+4);
// calculate the input angle
a1 = _mm_mul_pd(_mm_mul_pd(di1, di1), rate);
a2 = _mm_mul_pd(_mm_mul_pd(di2, di2), rate);
// update times for next
di1 = _mm_add_pd(di1, DFOUR);
di2 = _mm_add_pd(di2, DFOUR);
// reduce the angle to the range (-0.5, 0.5)
a1 = _mm_sub_pd(a1, _mm_sub_pd(_mm_add_pd(a1, roundVal), roundVal));
a2 = _mm_sub_pd(a2, _mm_sub_pd(_mm_add_pd(a2, roundVal), roundVal));
// convert pair of packed double into packed single
x = _mm_movelh_ps(_mm_cvtpd_ps(a1), _mm_cvtpd_ps(a2)); // 3 1 2 0
// square to the range [0, 0.25)
y = _mm_mul_ps(x, x);
// perform the initial polynomial approximations, Estrin's method
z = _mm_mul_ps(y, y);
s = _mm_mul_ps(_mm_add_ps(_mm_mul_ps(_mm_add_ps(_mm_mul_ps(y, SS4F),
SS3F),
z),
_mm_add_ps(_mm_mul_ps(y, SS2F),
SS1F)),
x);
c = _mm_add_ps(_mm_mul_ps(_mm_add_ps(_mm_mul_ps(y, CC3F),
CC2F),
z),
_mm_add_ps(_mm_mul_ps(y, CC1F),
ONE));
// perform first angle doubling
x = _mm_sub_ps(_mm_mul_ps(c, c), _mm_mul_ps(s, s));
y = _mm_mul_ps(_mm_mul_ps(s, c), TWO);
// calculate scaling factor to correct the magnitude
m = _mm_sub_ps(_mm_sub_ps(TWO, _mm_mul_ps(x, x)), _mm_mul_ps(y, y));
// perform second angle doubling
c = _mm_sub_ps(_mm_mul_ps(x, x), _mm_mul_ps(y, y));
s = _mm_mul_ps(_mm_mul_ps(y, x), TWO);
// correct the magnitude (final sine / cosine approximations)
c = _mm_mul_ps(c, m); // c3 c1 c2 c0
s = _mm_mul_ps(s, m);
// chirp the data
cd1 = _mm_moveldup_ps(c); // c1 c1 c0 c0
cd2 = _mm_movehdup_ps(c); // c3 c3 c2 c2
cd1 = _mm_mul_ps(cd1, d1); // c1.i1 c1.r1 c0.i0 c0.r0
cd2 = _mm_mul_ps(cd2, d2); // c3.i3 c3.r3 c2.i2 c2.r2
d1 = _mm_shuffle_ps(d1, d1, 0xb1);
d2 = _mm_shuffle_ps(d2, d2, 0xb1);
td1 = _mm_moveldup_ps(s);
td2 = _mm_movehdup_ps(s);
td1 = _mm_mul_ps(td1, d1);
td2 = _mm_mul_ps(td2, d2);
cd1 = _mm_addsub_ps(cd1, td1);
cd2 = _mm_addsub_ps(cd2, td2);
// store chirped values
_mm_stream_ps(cd, cd1);
_mm_stream_ps(cd+4, cd2);
}
// handle tail elements with scalar code
for (; i < ul_NumDataPoints; ++i) {
double angle = srate * i * i * 0.5;
double s = sin(angle);
double c = cos(angle);
float re = cx_DataArray[i][0];
float im = cx_DataArray[i][1];
cx_ChirpDataArray[i][0] = re * c - im * s;
cx_ChirpDataArray[i][1] = re * s + im * c;
}
analysis_state.FLOP_counter+=12.0*ul_NumDataPoints;
#ifdef USE_MANUAL_CALLSTACK
call_stack.exit();
#endif
return 0;
}
void process_sse2(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid,
const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out)
{
const dt_iop_graduatednd_data_t *data = (dt_iop_graduatednd_data_t *)piece->data;
const int ch = piece->colors;
const int ix = (roi_in->x);
const int iy = (roi_in->y);
const float iw = piece->buf_in.width * roi_out->scale;
const float ih = piece->buf_in.height * roi_out->scale;
const float hw = iw / 2.0;
const float hh = ih / 2.0;
const float hw_inv = 1.0 / hw;
const float hh_inv = 1.0 / hh;
const float v = (-data->rotation / 180) * M_PI;
const float sinv = sin(v);
const float cosv = cos(v);
const float filter_radie = sqrt((hh * hh) + (hw * hw)) / hh;
const float offset = data->offset / 100.0 * 2;
float color[3];
hsl2rgb(color, data->hue, data->saturation, 0.5);
if(data->density < 0)
for(int l = 0; l < 3; l++) color[l] = 1.0 - color[l];
#if 1
const float filter_compression = 1.0 / filter_radie
/ (1.0 - (0.5 + (data->compression / 100.0) * 0.9 / 2.0)) * 0.5;
#else
const float compression = data->compression / 100.0f;
const float t = 1.0f - .8f / (.8f + compression);
const float c = 1.0f + 1000.0f * powf(4.0, compression);
#endif
if(data->density > 0)
{
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(roi_out, color, data, ivoid, ovoid) schedule(static)
#endif
for(int y = 0; y < roi_out->height; y++)
{
size_t k = (size_t)roi_out->width * y * ch;
const float *in = (float *)ivoid + k;
float *out = (float *)ovoid + k;
float length = (sinv * (-1.0 + ix * hw_inv) - cosv * (-1.0 + (iy + y) * hh_inv) - 1.0 + offset)
* filter_compression;
const float length_inc = sinv * hw_inv * filter_compression;
__m128 c = _mm_set_ps(0, color[2], color[1], color[0]);
__m128 c1 = _mm_sub_ps(_mm_set1_ps(1.0f), c);
for(int x = 0; x < roi_out->width; x++, in += ch, out += ch)
{
#if 1
// !!! approximation is ok only when highest density is 8
// for input x = (data->density * CLIP( 0.5+length ), calculate 2^x as (e^(ln2*x/8))^8
// use exp2f approximation to calculate e^(ln2*x/8)
// in worst case - density==8,CLIP(0.5-length) == 1.0 it gives 0.6% of error
const float t = 0.693147181f /* ln2 */ * (data->density * CLIP(0.5f + length) / 8.0f);
float d1 = t * t * 0.5f;
float d2 = d1 * t * 0.333333333f;
float d3 = d2 * t * 0.25f;
float d = 1 + t + d1 + d2 + d3; /* taylor series for e^x till x^4 */
// printf("%d %d %f\n",y,x,d);
__m128 density = _mm_set1_ps(d);
density = _mm_mul_ps(density, density);
density = _mm_mul_ps(density, density);
density = _mm_mul_ps(density, density);
#else
// use fair exp2f
__m128 density = _mm_set1_ps(exp2f(data->density * CLIP(0.5f + length)));
#endif
/* max(0,in / (c + (1-c)*density)) */
_mm_stream_ps(out, _mm_max_ps(_mm_set1_ps(0.0f),
_mm_div_ps(_mm_load_ps(in), _mm_add_ps(c, _mm_mul_ps(c1, density)))));
length += length_inc;
}
}
}
else
{
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(roi_out, color, data, ivoid, ovoid) schedule(static)
#endif
for(int y = 0; y < roi_out->height; y++)
{
size_t k = (size_t)roi_out->width * y * ch;
const float *in = (float *)ivoid + k;
float *out = (float *)ovoid + k;
float length = (sinv * (-1.0f + ix * hw_inv) - cosv * (-1.0f + (iy + y) * hh_inv) - 1.0f + offset)
* filter_compression;
const float length_inc = sinv * hw_inv * filter_compression;
__m128 c = _mm_set_ps(0, color[2], color[1], color[0]);
__m128 c1 = _mm_sub_ps(_mm_set1_ps(1.0f), c);
for(int x = 0; x < roi_out->width; x++, in += ch, out += ch)
{
#if 1
// !!! approximation is ok only when lowest density is -8
// for input x = (-data->density * CLIP( 0.5-length ), calculate 2^x as (e^(ln2*x/8))^8
// use exp2f approximation to calculate e^(ln2*x/8)
// in worst case - density==-8,CLIP(0.5-length) == 1.0 it gives 0.6% of error
const float t = 0.693147181f /* ln2 */ * (-data->density * CLIP(0.5f - length) / 8.0f);
float d1 = t * t * 0.5f;
float d2 = d1 * t * 0.333333333f;
float d3 = d2 * t * 0.25f;
float d = 1 + t + d1 + d2 + d3; /* taylor series for e^x till x^4 */
__m128 density = _mm_set1_ps(d);
density = _mm_mul_ps(density, density);
density = _mm_mul_ps(density, density);
density = _mm_mul_ps(density, density);
#else
__m128 density = _mm_set1_ps(exp2f(-data->density * CLIP(0.5f - length)));
#endif
/* max(0,in * (c + (1-c)*density)) */
_mm_stream_ps(out, _mm_max_ps(_mm_set1_ps(0.0f),
_mm_mul_ps(_mm_load_ps(in), _mm_add_ps(c, _mm_mul_ps(c1, density)))));
length += length_inc;
}
}
}
_mm_sfence();
if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height);
}
void process(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid,
const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out)
{
const int filters = dt_image_filter(&piece->pipe->image);
uint8_t (*const xtrans)[6] = self->dev->image_storage.xtrans;
dt_iop_temperature_data_t *d = (dt_iop_temperature_data_t *)piece->data;
if(!dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) && filters == 9u)
{ // xtrans float mosaiced
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid, d) schedule(static)
#endif
for(int j = 0; j < roi_out->height; j++)
{
const float *in = ((float *)ivoid) + (size_t)j * roi_out->width;
float *out = ((float *)ovoid) + (size_t)j * roi_out->width;
for(int i = 0; i < roi_out->width; i++, out++, in++)
*out = *in * d->coeffs[FCxtrans(j, i, roi_out, xtrans)];
}
}
else if(!dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) && filters)
{ // bayer float mosaiced
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid, d) schedule(static)
#endif
for(int j = 0; j < roi_out->height; j++)
{
const float *in = ((float *)ivoid) + (size_t)j * roi_out->width;
float *out = ((float *)ovoid) + (size_t)j * roi_out->width;
int i = 0;
int alignment = ((4 - (j * roi_out->width & (4 - 1))) & (4 - 1));
// process unaligned pixels
for(; i < alignment; i++, out++, in++)
*out = *in * d->coeffs[FC(j + roi_out->y, i + roi_out->x, filters)];
const __m128 coeffs = _mm_set_ps(d->coeffs[FC(j + roi_out->y, roi_out->x + i + 3, filters)],
d->coeffs[FC(j + roi_out->y, roi_out->x + i + 2, filters)],
d->coeffs[FC(j + roi_out->y, roi_out->x + i + 1, filters)],
d->coeffs[FC(j + roi_out->y, roi_out->x + i, filters)]);
// process aligned pixels with SSE
for(; i < roi_out->width - (4 - 1); i += 4, in += 4, out += 4)
{
const __m128 input = _mm_load_ps(in);
const __m128 multiplied = _mm_mul_ps(input, coeffs);
_mm_stream_ps(out, multiplied);
}
// process the rest
for(; i < roi_out->width; i++, out++, in++)
*out = *in * d->coeffs[FC(j + roi_out->y, i + roi_out->x, filters)];
}
_mm_sfence();
}
else
{ // non-mosaiced
const int ch = piece->colors;
const __m128 coeffs = _mm_set_ps(1.0f, d->coeffs[2], d->coeffs[1], d->coeffs[0]);
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid, d) schedule(static)
#endif
for(int k = 0; k < roi_out->height; k++)
{
const float *in = ((float *)ivoid) + (size_t)ch * k * roi_out->width;
float *out = ((float *)ovoid) + (size_t)ch * k * roi_out->width;
for(int j = 0; j < roi_out->width; j++, in += ch, out += ch)
{
const __m128 input = _mm_load_ps(in);
const __m128 multiplied = _mm_mul_ps(input, coeffs);
_mm_stream_ps(out, multiplied);
}
}
_mm_sfence();
if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height);
}
for(int k = 0; k < 3; k++)
piece->pipe->processed_maximum[k] = d->coeffs[k] * piece->pipe->processed_maximum[k];
}
void process(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid,
const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out)
{
float *in;
float *out;
dt_iop_zonesystem_gui_data_t *g = NULL;
dt_iop_zonesystem_data_t *data = (dt_iop_zonesystem_data_t *)piece->data;
const int width = roi_out->width;
const int height = roi_out->height;
if(self->dev->gui_attached && piece->pipe->type == DT_DEV_PIXELPIPE_PREVIEW)
{
g = (dt_iop_zonesystem_gui_data_t *)self->gui_data;
dt_pthread_mutex_lock(&g->lock);
if(g->in_preview_buffer == NULL || g->out_preview_buffer == NULL || g->preview_width != width
|| g->preview_height != height)
{
g_free(g->in_preview_buffer);
g_free(g->out_preview_buffer);
g->in_preview_buffer = g_malloc_n((size_t)width * height, sizeof(guchar));
g->out_preview_buffer = g_malloc_n((size_t)width * height, sizeof(guchar));
g->preview_width = width;
g->preview_height = height;
}
dt_pthread_mutex_unlock(&g->lock);
}
/* calculate zonemap */
const int size = data->size;
float zonemap[MAX_ZONE_SYSTEM_SIZE] = { -1 };
_iop_zonesystem_calculate_zonemap(data, zonemap);
const int ch = piece->colors;
/* process the image */
in = (float *)ivoid;
out = (float *)ovoid;
const float rzscale = (size - 1) / 100.0f;
float zonemap_offset[MAX_ZONE_SYSTEM_SIZE] = { -1 };
float zonemap_scale[MAX_ZONE_SYSTEM_SIZE] = { -1 };
// precompute scale and offset
for(int k = 0; k < size - 1; k++) zonemap_scale[k] = (zonemap[k + 1] - zonemap[k]) * (size - 1);
for(int k = 0; k < size - 1; k++) zonemap_offset[k] = 100.0f * ((k + 1) * zonemap[k] - k * zonemap[k + 1]);
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(in, out, zonemap_scale, zonemap_offset) schedule(static)
#endif
for(int j = 0; j < height; j++)
for(int i = 0; i < width; i++)
{
/* remap lightness into zonemap and apply lightness */
const float *inp = in + ch * ((size_t)j * width + i);
float *outp = out + ch * ((size_t)j * width + i);
const int rz = CLAMPS(inp[0] * rzscale, 0, size - 2); // zone index
const float zs = ((rz > 0) ? (zonemap_offset[rz] / inp[0]) : 0) + zonemap_scale[rz];
_mm_stream_ps(outp, _mm_mul_ps(_mm_load_ps(inp), _mm_set1_ps(zs)));
}
_mm_sfence();
if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, width, height);
/* if gui and have buffer lets gaussblur and fill buffer with zone indexes */
if(self->dev->gui_attached && g && g->in_preview_buffer && g->out_preview_buffer)
{
float Lmax[] = { 100.0f };
float Lmin[] = { 0.0f };
/* setup gaussian kernel */
const int radius = 8;
const float sigma = 2.5 * (radius * roi_in->scale / piece->iscale);
dt_gaussian_t *gauss = dt_gaussian_init(width, height, 1, Lmax, Lmin, sigma, DT_IOP_GAUSSIAN_ZERO);
float *tmp = g_malloc_n((size_t)width * height, sizeof(float));
if(gauss && tmp)
{
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(ivoid, tmp) schedule(static)
#endif
for(size_t k = 0; k < (size_t)width * height; k++) tmp[k] = ((float *)ivoid)[ch * k];
dt_gaussian_blur(gauss, tmp, tmp);
/* create zonemap preview for input */
dt_pthread_mutex_lock(&g->lock);
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(tmp, g) schedule(static)
#endif
for(size_t k = 0; k < (size_t)width * height; k++)
{
g->in_preview_buffer[k] = CLAMPS(tmp[k] * (size - 1) / 100.0f, 0, size - 2);
}
dt_pthread_mutex_unlock(&g->lock);
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(ovoid, tmp) schedule(static)
#endif
for(size_t k = 0; k < (size_t)width * height; k++) tmp[k] = ((float *)ovoid)[ch * k];
dt_gaussian_blur(gauss, tmp, tmp);
/* create zonemap preview for output */
dt_pthread_mutex_lock(&g->lock);
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(tmp, g) schedule(static)
#endif
for(size_t k = 0; k < (size_t)width * height; k++)
{
g->out_preview_buffer[k] = CLAMPS(tmp[k] * (size - 1) / 100.0f, 0, size - 2);
}
dt_pthread_mutex_unlock(&g->lock);
}
g_free(tmp);
if(gauss) dt_gaussian_free(gauss);
}
}
static void gui_update_from_coeffs(dt_iop_module_t *self)
{
dt_iop_invert_gui_data_t *g = (dt_iop_invert_gui_data_t *)self->gui_data;
dt_iop_invert_params_t *p = (dt_iop_invert_params_t *)self->params;
GdkRGBA color = (GdkRGBA){.red = p->color[0], .green = p->color[1], .blue = p->color[2], .alpha = 1.0 };
const dt_image_t *img = &self->dev->image_storage;
if(img->flags & DT_IMAGE_4BAYER)
{
float rgb[4];
for(int k = 0; k < 4; k++) rgb[k] = p->color[k];
dt_colorspaces_cygm_to_rgb(rgb, 1, g->CAM_to_RGB);
color.red = rgb[0];
color.green = rgb[1];
color.blue = rgb[2];
}
gtk_color_chooser_set_rgba(GTK_COLOR_CHOOSER(g->colorpicker), &color);
}
static gboolean draw(GtkWidget *widget, cairo_t *cr, dt_iop_module_t *self)
{
if(darktable.gui->reset) return FALSE;
if(self->picked_color_max[0] < 0.0f) return FALSE;
if(self->request_color_pick == DT_REQUEST_COLORPICK_OFF) return FALSE;
static float old[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
const float *grayrgb = self->picked_color;
if(grayrgb[0] == old[0] && grayrgb[1] == old[1] && grayrgb[2] == old[2] && grayrgb[3] == old[3]) return FALSE;
for(int k = 0; k < 4; k++) old[k] = grayrgb[k];
dt_iop_invert_params_t *p = self->params;
for(int k = 0; k < 4; k++) p->color[k] = grayrgb[k];
darktable.gui->reset = 1;
gui_update_from_coeffs(self);
darktable.gui->reset = 0;
dt_dev_add_history_item(darktable.develop, self, TRUE);
return FALSE;
}
static void colorpicker_callback(GtkColorButton *widget, dt_iop_module_t *self)
{
if(self->dt->gui->reset) return;
dt_iop_invert_gui_data_t *g = (dt_iop_invert_gui_data_t *)self->gui_data;
dt_iop_invert_params_t *p = (dt_iop_invert_params_t *)self->params;
// turn off the other color picker so that this tool actually works ...
gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(g->picker), FALSE);
GdkRGBA c;
gtk_color_chooser_get_rgba(GTK_COLOR_CHOOSER(widget), &c);
p->color[0] = c.red;
p->color[1] = c.green;
p->color[2] = c.blue;
const dt_image_t *img = &self->dev->image_storage;
if(img->flags & DT_IMAGE_4BAYER)
{
dt_colorspaces_rgb_to_cygm(p->color, 1, g->RGB_to_CAM);
}
dt_dev_add_history_item(darktable.develop, self, TRUE);
}
void process(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, const void *const ivoid,
void *const ovoid, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out)
{
const dt_iop_invert_data_t *const d = (dt_iop_invert_data_t *)piece->data;
const float *const m = piece->pipe->dsc.processed_maximum;
const float film_rgb_f[4]
= { d->color[0] * m[0], d->color[1] * m[1], d->color[2] * m[2], d->color[3] * m[3] };
// FIXME: it could be wise to make this a NOP when picking colors. not sure about that though.
// if(self->request_color_pick){
// do nothing
// }
const uint32_t filters = piece->pipe->dsc.filters;
const uint8_t(*const xtrans)[6] = (const uint8_t(*const)[6])piece->pipe->dsc.xtrans;
const float *const in = (const float *const)ivoid;
float *const out = (float *const)ovoid;
if(filters == 9u)
{ // xtrans float mosaiced
#ifdef _OPENMP
#pragma omp parallel for SIMD() default(none) schedule(static) collapse(2)
#endif
for(int j = 0; j < roi_out->height; j++)
{
for(int i = 0; i < roi_out->width; i++)
{
const size_t p = (size_t)j * roi_out->width + i;
out[p] = CLAMP(film_rgb_f[FCxtrans(j, i, roi_out, xtrans)] - in[p], 0.0f, 1.0f);
}
}
for(int k = 0; k < 4; k++) piece->pipe->dsc.processed_maximum[k] = 1.0f;
}
else if(filters)
{ // bayer float mosaiced
#ifdef _OPENMP
#pragma omp parallel for SIMD() default(none) schedule(static) collapse(2)
#endif
for(int j = 0; j < roi_out->height; j++)
{
for(int i = 0; i < roi_out->width; i++)
{
const size_t p = (size_t)j * roi_out->width + i;
out[p] = CLAMP(film_rgb_f[FC(j + roi_out->y, i + roi_out->x, filters)] - in[p], 0.0f, 1.0f);
}
}
for(int k = 0; k < 4; k++) piece->pipe->dsc.processed_maximum[k] = 1.0f;
}
else
{ // non-mosaiced
const int ch = piece->colors;
#ifdef _OPENMP
#pragma omp parallel for SIMD() default(none) schedule(static) collapse(2)
#endif
for(size_t k = 0; k < (size_t)ch * roi_out->width * roi_out->height; k += ch)
{
for(int c = 0; c < 3; c++)
{
const size_t p = (size_t)k + c;
out[p] = d->color[c] - in[p];
}
}
if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height);
}
}
#if defined(__SSE__)
void process_sse2(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, const void *const ivoid,
void *const ovoid, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out)
{
dt_iop_invert_data_t *d = (dt_iop_invert_data_t *)piece->data;
const float *const m = piece->pipe->dsc.processed_maximum;
const float film_rgb_f[4]
= { d->color[0] * m[0], d->color[1] * m[1], d->color[2] * m[2], d->color[3] * m[3] };
// FIXME: it could be wise to make this a NOP when picking colors. not sure about that though.
// if(self->request_color_pick){
// do nothing
// }
const uint32_t filters = piece->pipe->dsc.filters;
const uint8_t(*const xtrans)[6] = (const uint8_t(*const)[6])piece->pipe->dsc.xtrans;
if(filters == 9u)
{ // xtrans float mosaiced
const __m128 val_min = _mm_setzero_ps();
const __m128 val_max = _mm_set1_ps(1.0f);
#ifdef _OPENMP
#pragma omp parallel for default(none) schedule(static)
#endif
for(int j = 0; j < roi_out->height; j++)
{
const float *in = ((float *)ivoid) + (size_t)j * roi_out->width;
float *out = ((float *)ovoid) + (size_t)j * roi_out->width;
int i = 0;
int alignment = ((4 - (j * roi_out->width & (4 - 1))) & (4 - 1));
// process unaligned pixels
for(; i < alignment && i < roi_out->width; i++, out++, in++)
*out = CLAMP(film_rgb_f[FCxtrans(j, i, roi_out, xtrans)] - *in, 0.0f, 1.0f);
const __m128 film[3] = { _mm_set_ps(film_rgb_f[FCxtrans(j, i + 3, roi_out, xtrans)],
film_rgb_f[FCxtrans(j, i + 2, roi_out, xtrans)],
film_rgb_f[FCxtrans(j, i + 1, roi_out, xtrans)],
film_rgb_f[FCxtrans(j, i + 0, roi_out, xtrans)]),
_mm_set_ps(film_rgb_f[FCxtrans(j, i + 7, roi_out, xtrans)],
film_rgb_f[FCxtrans(j, i + 6, roi_out, xtrans)],
film_rgb_f[FCxtrans(j, i + 5, roi_out, xtrans)],
film_rgb_f[FCxtrans(j, i + 4, roi_out, xtrans)]),
_mm_set_ps(film_rgb_f[FCxtrans(j, i + 11, roi_out, xtrans)],
film_rgb_f[FCxtrans(j, i + 10, roi_out, xtrans)],
film_rgb_f[FCxtrans(j, i + 9, roi_out, xtrans)],
film_rgb_f[FCxtrans(j, i + 8, roi_out, xtrans)]) };
// process aligned pixels with SSE
for(int c = 0; c < 3 && i < roi_out->width - (4 - 1); c++, i += 4, in += 4, out += 4)
{
__m128 v;
v = _mm_load_ps(in);
v = _mm_sub_ps(film[c], v);
v = _mm_min_ps(v, val_max);
v = _mm_max_ps(v, val_min);
_mm_stream_ps(out, v);
}
// process the rest
for(; i < roi_out->width; i++, out++, in++)
*out = CLAMP(film_rgb_f[FCxtrans(j, i, roi_out, xtrans)] - *in, 0.0f, 1.0f);
}
_mm_sfence();
for(int k = 0; k < 4; k++) piece->pipe->dsc.processed_maximum[k] = 1.0f;
}
else if(filters)
{ // bayer float mosaiced
const __m128 val_min = _mm_setzero_ps();
const __m128 val_max = _mm_set1_ps(1.0f);
#ifdef _OPENMP
#pragma omp parallel for default(none) schedule(static)
#endif
for(int j = 0; j < roi_out->height; j++)
{
const float *in = ((float *)ivoid) + (size_t)j * roi_out->width;
float *out = ((float *)ovoid) + (size_t)j * roi_out->width;
int i = 0;
int alignment = ((4 - (j * roi_out->width & (4 - 1))) & (4 - 1));
// process unaligned pixels
for(; i < alignment && i < roi_out->width; i++, out++, in++)
*out = CLAMP(film_rgb_f[FC(j + roi_out->y, i + roi_out->x, filters)] - *in, 0.0f, 1.0f);
const __m128 film = _mm_set_ps(film_rgb_f[FC(j + roi_out->y, roi_out->x + i + 3, filters)],
film_rgb_f[FC(j + roi_out->y, roi_out->x + i + 2, filters)],
film_rgb_f[FC(j + roi_out->y, roi_out->x + i + 1, filters)],
film_rgb_f[FC(j + roi_out->y, roi_out->x + i, filters)]);
// process aligned pixels with SSE
for(; i < roi_out->width - (4 - 1); i += 4, in += 4, out += 4)
{
const __m128 input = _mm_load_ps(in);
const __m128 subtracted = _mm_sub_ps(film, input);
_mm_stream_ps(out, _mm_max_ps(_mm_min_ps(subtracted, val_max), val_min));
}
// process the rest
for(; i < roi_out->width; i++, out++, in++)
*out = CLAMP(film_rgb_f[FC(j + roi_out->y, i + roi_out->x, filters)] - *in, 0.0f, 1.0f);
}
_mm_sfence();
for(int k = 0; k < 4; k++) piece->pipe->dsc.processed_maximum[k] = 1.0f;
}
else
{ // non-mosaiced
const int ch = piece->colors;
const __m128 film = _mm_set_ps(1.0f, d->color[2], d->color[1], d->color[0]);
#ifdef _OPENMP
#pragma omp parallel for default(none) schedule(static)
#endif
for(int k = 0; k < roi_out->height; k++)
{
const float *in = ((float *)ivoid) + (size_t)ch * k * roi_out->width;
float *out = ((float *)ovoid) + (size_t)ch * k * roi_out->width;
for(int j = 0; j < roi_out->width; j++, in += ch, out += ch)
{
const __m128 input = _mm_load_ps(in);
const __m128 subtracted = _mm_sub_ps(film, input);
_mm_stream_ps(out, subtracted);
}
}
_mm_sfence();
if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height);
}
}
/* Fast remote SCI copy for systems with write-combining enabled.
This is the version using SSE instructions to copy 128 Byte blocks,
and flushes after 32 Byte. */
void _mpid_smi_sse32_memcpy(void *dest, const void *src, size_t size)
{
char* a = (char*) src;
char* b = (char*) dest;
size_t j = 0;
__m128 xmm[8];
/* Align the destination to a 64 Byte boundary */
for(; (j < size) && (((size_t) &b[j]) % 64 != 0); j++)
((char*) b)[j] = ((char*) a)[j];
// Loads two cache lines of data to a location closer to the processor.
_mm_prefetch(a+j, _MM_HINT_NTA);
_mm_prefetch(a+j+64, _MM_HINT_NTA);
/* copy 128 byte per loop */
for (; (j+128) < size; j+=128)
{
// Loads two cache lines of data to a location closer to the processor.
_mm_prefetch(a+j+128, _MM_HINT_NTA);
_mm_prefetch(a+j+192, _MM_HINT_NTA);
/* load 128 Byte into xmm register */
xmm[0] = _mm_load_ps((float*) &a[j]);
xmm[1] = _mm_load_ps((float*) &a[j+16]);
xmm[2] = _mm_load_ps((float*) &a[j+32]);
xmm[3] = _mm_load_ps((float*) &a[j+48]);
xmm[4] = _mm_load_ps((float*) &a[j+64]);
xmm[5] = _mm_load_ps((float*) &a[j+80]);
xmm[6] = _mm_load_ps((float*) &a[j+96]);
xmm[7] = _mm_load_ps((float*) &a[j+112]);
/* store 32 byte */
_mm_stream_ps((float*) &b[j], xmm[0]);
_mm_stream_ps((float*) &b[j+16], xmm[1]);
/* flush the write-combine buffer */
_mm_sfence();
/* store 32 byte */
_mm_stream_ps((float*) &b[j+32], xmm[2]);
_mm_stream_ps((float*) &b[j+48], xmm[3]);
/* flush the write-combine buffer */
_mm_sfence();
/* store 32 byte */
_mm_stream_ps((float*) &b[j+64], xmm[4]);
_mm_stream_ps((float*) &b[j+80], xmm[5]);
/* flush the write-combine buffer */
_mm_sfence();
/* store 32 byte */
_mm_stream_ps((float*) &b[j+96], xmm[6]);
_mm_stream_ps((float*) &b[j+112], xmm[7]);
/* flush the write-combine buffer */
_mm_sfence();
}
/* copy tail */
for(; j<size; j++)
((char*) b)[j] = ((char*) a)[j];
}
void
dt_interpolation_resample(
const struct dt_interpolation* itor,
float *out,
const dt_iop_roi_t* const roi_out,
const int32_t out_stride,
const float* const in,
const dt_iop_roi_t* const roi_in,
const int32_t in_stride)
{
int* hindex = NULL;
int* hlength = NULL;
float* hkernel = NULL;
int* vindex = NULL;
int* vlength = NULL;
float* vkernel = NULL;
int* vmeta = NULL;
int r;
debug_info(
"resampling %p (%dx%d@%dx%d scale %f) -> %p (%dx%d@%dx%d scale %f)\n",
in,
roi_in->width, roi_in->height, roi_in->x, roi_in->y, roi_in->scale,
out,
roi_out->width, roi_out->height, roi_out->x, roi_out->y, roi_out->scale);
// Fast code path for 1:1 copy, only cropping area can change
if (roi_out->scale == 1.f) {
const int x0 = roi_out->x*4*sizeof(float);
const int l = roi_out->width*4*sizeof(float);
#if DEBUG_RESAMPLING_TIMING
int64_t ts_resampling = getts();
#endif
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(out)
#endif
for (int y=0; y<roi_out->height; y++) {
float* i = (float*)((char*)in + in_stride*(y + roi_out->y) + x0);
float* o = (float*)((char*)out + out_stride*y);
memcpy(o, i, l);
}
#if DEBUG_RESAMPLING_TIMING
ts_resampling = getts() - ts_resampling;
fprintf(stderr, "resampling %p plan:0us resampling:%"PRId64"us\n", in, ts_resampling);
#endif
// All done, so easy case
return;
}
// Generic non 1:1 case... much more complicated :D
#if DEBUG_RESAMPLING_TIMING
int64_t ts_plan = getts();
#endif
// Prepare resampling plans once and for all
r = prepare_resampling_plan(itor, roi_in->width, roi_in->x, roi_out->width, roi_out->x, roi_out->scale, &hlength, &hkernel, &hindex, NULL);
if (r) {
goto exit;
}
r = prepare_resampling_plan(itor, roi_in->height, roi_in->y, roi_out->height, roi_out->y, roi_out->scale, &vlength, &vkernel, &vindex, &vmeta);
if (r) {
goto exit;
}
#if DEBUG_RESAMPLING_TIMING
ts_plan = getts() - ts_plan;
#endif
#if DEBUG_RESAMPLING_TIMING
int64_t ts_resampling = getts();
#endif
// Process each output line
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(out, hindex, hlength, hkernel, vindex, vlength, vkernel, vmeta)
#endif
for (int oy=0; oy<roi_out->height; oy++) {
// Initialize column resampling indexes
int vlidx = vmeta[3*oy + 0]; // V(ertical) L(ength) I(n)d(e)x
int vkidx = vmeta[3*oy + 1]; // V(ertical) K(ernel) I(n)d(e)x
int viidx = vmeta[3*oy + 2]; // V(ertical) I(ndex) I(n)d(e)x
// Initialize row resampling indexes
int hlidx = 0; // H(orizontal) L(ength) I(n)d(e)x
int hkidx = 0; // H(orizontal) K(ernel) I(n)d(e)x
int hiidx = 0; // H(orizontal) I(ndex) I(n)d(e)x
// Number of lines contributing to the output line
int vl = vlength[vlidx++]; // V(ertical) L(ength)
// Process each output column
for (int ox=0; ox < roi_out->width; ox++) {
debug_extra("output %p [% 4d % 4d]\n", out, ox, oy);
// This will hold the resulting pixel
__m128 vs = _mm_setzero_ps();
// Number of horizontal samples contributing to the output
int hl = hlength[hlidx++]; // H(orizontal) L(ength)
for (int iy=0; iy < vl; iy++) {
// This is our input line
const float* i = (float*)((char*)in + in_stride*vindex[viidx++]);
__m128 vhs = _mm_setzero_ps();
for (int ix=0; ix< hl; ix++) {
// Apply the precomputed filter kernel
int baseidx = hindex[hiidx++]*4;
float htap = hkernel[hkidx++];
__m128 vhtap = _mm_set_ps1(htap);
vhs = _mm_add_ps(vhs, _mm_mul_ps(*(__m128*)&i[baseidx], vhtap));
}
// Accumulate contribution from this line
float vtap = vkernel[vkidx++];
__m128 vvtap = _mm_set_ps1(vtap);
vs = _mm_add_ps(vs, _mm_mul_ps(vhs, vvtap));
// Reset horizontal resampling context
hkidx -= hl;
hiidx -= hl;
}
// Output pixel is ready
float* o = (float*)((char*)out + oy*out_stride + ox*4*sizeof(float));
_mm_stream_ps(o, vs);
// Reset vertical resampling context
viidx -= vl;
vkidx -= vl;
// Progress in horizontal context
hiidx += hl;
hkidx += hl;
}
// Progress in vertical context
viidx += vl;
vkidx += vl;
}
_mm_sfence();
#if DEBUG_RESAMPLING_TIMING
ts_resampling = getts() - ts_resampling;
fprintf(stderr, "resampling %p plan:%"PRId64"us resampling:%"PRId64"us\n", in, ts_plan, ts_resampling);
#endif
exit:
/* Free the resampling plans. It's nasty to optimize allocs like that, but
* it simplifies the code :-D. The length array is in fact the only memory
* allocated. */
free(hlength);
free(vlength);
}
void
process (struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, const void * const ivoid, void *ovoid, const dt_iop_roi_t *roi_in, const dt_iop_roi_t * const roi_out)
{
dt_develop_t *dev = self->dev;
const int ch = piece->colors;
// FIXME: turn off the module instead?
if(!dev->overexposed.enabled || !dev->gui_attached)
{
memcpy(ovoid, ivoid, (size_t)roi_out->width*roi_out->height*sizeof(float)*ch);
return;
}
const __m128 upper = _mm_set_ps(FLT_MAX,
dev->overexposed.upper / 100.0f,
dev->overexposed.upper / 100.0f,
dev->overexposed.upper / 100.0f);
const __m128 lower = _mm_set_ps(FLT_MAX,
dev->overexposed.lower / 100.0f,
dev->overexposed.lower / 100.0f,
dev->overexposed.lower / 100.0f);
const int colorscheme = dev->overexposed.colorscheme;
const __m128 upper_color = _mm_load_ps(dt_iop_overexposed_colors[colorscheme][0]);
const __m128 lower_color = _mm_load_ps(dt_iop_overexposed_colors[colorscheme][1]);
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(ovoid) schedule(static)
#endif
for(int k=0; k<roi_out->height; k++)
{
const float *in = ((float *)ivoid) + (size_t)ch*k*roi_out->width;
float *out = ((float *)ovoid) + (size_t)ch*k*roi_out->width;
for (int j=0; j<roi_out->width; j++,in+=4,out+=4)
{
const __m128 pixel = _mm_load_ps(in);
__m128 isoe = _mm_cmpge_ps(pixel, upper);
isoe = _mm_or_ps(_mm_unpacklo_ps(isoe, isoe), _mm_unpackhi_ps(isoe, isoe));
isoe = _mm_or_ps(_mm_unpacklo_ps(isoe, isoe), _mm_unpackhi_ps(isoe, isoe));
__m128 isue = _mm_cmple_ps(pixel, lower);
isue = _mm_and_ps(_mm_unpacklo_ps(isue, isue), _mm_unpackhi_ps(isue, isue));
isue = _mm_and_ps(_mm_unpacklo_ps(isue, isue), _mm_unpackhi_ps(isue, isue));
__m128 result = _mm_or_ps(_mm_andnot_ps(isoe, pixel),
_mm_and_ps(isoe, upper_color));
result = _mm_or_ps(_mm_andnot_ps(isue, result),
_mm_and_ps(isue, lower_color));
_mm_stream_ps(out, result);
}
}
_mm_sfence();
if(piece->pipe->mask_display)
dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height);
}
// =============================================================================
//
// sse3_vChirpData
// version by: Alex Kan
// http://tbp.berkeley.edu/~alexkan/seti/
//
int sse3_ChirpData_ak(
sah_complex * cx_DataArray,
sah_complex * cx_ChirpDataArray,
int chirp_rate_ind,
double chirp_rate,
int ul_NumDataPoints,
double sample_rate
) {
int i;
#ifdef USE_MANUAL_CALLSTACK
call_stack.enter("sse3_ChirpData_ak()");
#endif
if (chirp_rate_ind == 0) {
memcpy(cx_ChirpDataArray, cx_DataArray, (int)ul_NumDataPoints * sizeof(sah_complex) );
#ifdef USE_MANUAL_CALLSTACK
call_stack.exit();
#endif
return 0;
}
int vEnd;
double srate = chirp_rate * 0.5 / (sample_rate * sample_rate);
__m128d rate = _mm_set1_pd(chirp_rate * 0.5 / (sample_rate * sample_rate));
__m128d roundVal = _mm_set1_pd(srate >= 0.0 ? TWO_TO_52 : -TWO_TO_52);
// main vectorised loop
vEnd = ul_NumDataPoints - (ul_NumDataPoints & 3);
for (i = 0; i < vEnd; i += 4) {
const float *data = (const float *) (cx_DataArray + i);
float *chirped = (float *) (cx_ChirpDataArray + i);
__m128d di = _mm_set1_pd(i);
__m128d a1 = _mm_add_pd(_mm_set_pd(1.0, 0.0), di);
__m128d a2 = _mm_add_pd(_mm_set_pd(3.0, 2.0), di);
__m128 d1, d2;
__m128 cd1, cd2;
__m128 td1, td2;
__m128 x;
__m128 y;
__m128 s;
__m128 c;
__m128 m;
// load the signal to be chirped
prefetchnta((const void *)( data+32 ));
d1 = _mm_load_ps(data);
d2 = _mm_load_ps(data+4);
// calculate the input angle
a1 = _mm_mul_pd(_mm_mul_pd(a1, a1), rate);
a2 = _mm_mul_pd(_mm_mul_pd(a2, a2), rate);
// reduce the angle to the range (-0.5, 0.5)
a1 = _mm_sub_pd(a1, _mm_sub_pd(_mm_add_pd(a1, roundVal), roundVal));
a2 = _mm_sub_pd(a2, _mm_sub_pd(_mm_add_pd(a2, roundVal), roundVal));
// convert pair of packed double into packed single
x = _mm_movelh_ps(_mm_cvtpd_ps(a1), _mm_cvtpd_ps(a2));
// square to the range [0, 0.25)
y = _mm_mul_ps(x, x);
// perform the initial polynomial approximations
s = _mm_mul_ps(_mm_add_ps(_mm_mul_ps(_mm_add_ps(_mm_mul_ps(_mm_add_ps(_mm_mul_ps(y, SS4),
SS3),
y),
SS2),
y),
SS1),
x);
c = _mm_add_ps(_mm_mul_ps(_mm_add_ps(_mm_mul_ps(_mm_add_ps(_mm_mul_ps(y, CC3),
CC2),
y),
CC1),
y),
ONE);
// perform first angle doubling
x = _mm_sub_ps(_mm_mul_ps(c, c), _mm_mul_ps(s, s));
y = _mm_mul_ps(_mm_mul_ps(s, c), TWO);
// calculate scaling factor to correct the magnitude
// m1 = vec_nmsub(y1, y1, vec_nmsub(x1, x1, TWO));
// m2 = vec_nmsub(y2, y2, vec_nmsub(x2, x2, TWO));
m = vec_recip3(_mm_add_ps(_mm_mul_ps(x, x), _mm_mul_ps(y, y)));
// perform second angle doubling
c = _mm_sub_ps(_mm_mul_ps(x, x), _mm_mul_ps(y, y));
s = _mm_mul_ps(_mm_mul_ps(y, x), TWO);
// correct the magnitude (final sine / cosine approximations)
s = _mm_mul_ps(s, m);
c = _mm_mul_ps(c, m);
// chirp the data
cd1 = _mm_shuffle_ps(c, c, 0x50);
cd2 = _mm_shuffle_ps(c, c, 0xfa);
cd1 = _mm_mul_ps(cd1, d1);
cd2 = _mm_mul_ps(cd2, d2);
d1 = _mm_shuffle_ps(d1, d1, 0xb1);
d2 = _mm_shuffle_ps(d2, d2, 0xb1);
td1 = _mm_shuffle_ps(s, s, 0x50);
td2 = _mm_shuffle_ps(s, s, 0xfa);
td1 = _mm_mul_ps(td1, d1);
td2 = _mm_mul_ps(td2, d2);
cd1 = _mm_addsub_ps(cd1, td1);
cd2 = _mm_addsub_ps(cd2, td2);
// store chirped values
_mm_stream_ps(chirped, cd1);
_mm_stream_ps(chirped+4, cd2);
}
_mm_sfence();
// handle tail elements with scalar code
for ( ; i < ul_NumDataPoints; ++i) {
double angle = srate * i * i * 0.5;
double s = sin(angle);
double c = cos(angle);
float re = cx_DataArray[i][0];
float im = cx_DataArray[i][1];
cx_ChirpDataArray[i][0] = re * c - im * s;
cx_ChirpDataArray[i][1] = re * s + im * c;
}
analysis_state.FLOP_counter+=12.0*ul_NumDataPoints;
#ifdef USE_MANUAL_CALLSTACK
call_stack.exit();
#endif
return 0;
}
void process_sse2(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, const void *const ivoid,
void *const ovoid, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out)
{
const dt_iop_colorout_data_t *const d = (dt_iop_colorout_data_t *)piece->data;
const int ch = piece->colors;
const int gamutcheck = (d->mode == DT_PROFILE_GAMUTCHECK);
if(d->type == DT_COLORSPACE_LAB)
{
memcpy(ovoid, ivoid, sizeof(float)*4*roi_out->width*roi_out->height);
}
else if(!isnan(d->cmatrix[0]))
{
// fprintf(stderr,"Using cmatrix codepath\n");
// convert to rgb using matrix
#ifdef _OPENMP
#pragma omp parallel for schedule(static) default(none)
#endif
for(int j = 0; j < roi_out->height; j++)
{
float *in = (float *)ivoid + (size_t)ch * roi_in->width * j;
float *out = (float *)ovoid + (size_t)ch * roi_out->width * j;
const __m128 m0 = _mm_set_ps(0.0f, d->cmatrix[6], d->cmatrix[3], d->cmatrix[0]);
const __m128 m1 = _mm_set_ps(0.0f, d->cmatrix[7], d->cmatrix[4], d->cmatrix[1]);
const __m128 m2 = _mm_set_ps(0.0f, d->cmatrix[8], d->cmatrix[5], d->cmatrix[2]);
for(int i = 0; i < roi_out->width; i++, in += ch, out += ch)
{
const __m128 xyz = dt_Lab_to_XYZ_SSE(_mm_load_ps(in));
const __m128 t
= _mm_add_ps(_mm_mul_ps(m0, _mm_shuffle_ps(xyz, xyz, _MM_SHUFFLE(0, 0, 0, 0))),
_mm_add_ps(_mm_mul_ps(m1, _mm_shuffle_ps(xyz, xyz, _MM_SHUFFLE(1, 1, 1, 1))),
_mm_mul_ps(m2, _mm_shuffle_ps(xyz, xyz, _MM_SHUFFLE(2, 2, 2, 2)))));
_mm_stream_ps(out, t);
}
}
_mm_sfence();
process_fastpath_apply_tonecurves(self, piece, ivoid, ovoid, roi_in, roi_out);
}
else
{
// fprintf(stderr,"Using xform codepath\n");
const __m128 outofgamutpixel = _mm_set_ps(0.0f, 1.0f, 1.0f, 0.0f);
#ifdef _OPENMP
#pragma omp parallel for schedule(static) default(none)
#endif
for(int k = 0; k < roi_out->height; k++)
{
const float *in = ((float *)ivoid) + (size_t)ch * k * roi_out->width;
float *out = ((float *)ovoid) + (size_t)ch * k * roi_out->width;
cmsDoTransform(d->xform, in, out, roi_out->width);
if(gamutcheck)
{
for(int j = 0; j < roi_out->width; j++, out += 4)
{
const __m128 pixel = _mm_load_ps(out);
__m128 ingamut = _mm_cmplt_ps(pixel, _mm_set_ps(-FLT_MAX, 0.0f, 0.0f, 0.0f));
ingamut = _mm_or_ps(_mm_unpacklo_ps(ingamut, ingamut), _mm_unpackhi_ps(ingamut, ingamut));
ingamut = _mm_or_ps(_mm_unpacklo_ps(ingamut, ingamut), _mm_unpackhi_ps(ingamut, ingamut));
const __m128 result
= _mm_or_ps(_mm_and_ps(ingamut, outofgamutpixel), _mm_andnot_ps(ingamut, pixel));
_mm_stream_ps(out, result);
}
}
}
_mm_sfence();
}
if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height);
}
void process (struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid, const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out)
{
float *in;
float *out;
dt_iop_zonesystem_gui_data_t *g = NULL;
dt_iop_zonesystem_data_t *data = (dt_iop_zonesystem_data_t*)piece->data;
guchar *buffer = NULL;
if( self->dev->gui_attached && piece->pipe->type == DT_DEV_PIXELPIPE_PREVIEW )
{
g = (dt_iop_zonesystem_gui_data_t *)self->gui_data;
dt_pthread_mutex_lock(&g->lock);
if(g->preview_buffer)
g_free (g->preview_buffer);
buffer = g->preview_buffer = g_malloc (roi_in->width*roi_in->height);
g->preview_width=roi_out->width;
g->preview_height=roi_out->height;
}
/* calculate zonemap */
const int size = data->size;
float zonemap[MAX_ZONE_SYSTEM_SIZE]= {-1};
_iop_zonesystem_calculate_zonemap (data, zonemap);
const int ch = piece->colors;
/* if gui and have buffer lets gaussblur and fill buffer with zone indexes */
if( self->dev->gui_attached && g && buffer)
{
/* setup gaussian kernel */
const int radius = 8;
const float _r = ceilf(radius * roi_in->scale / piece->iscale);
const int rad = MIN(radius, _r);
const int wd = 2*rad+1;
float mat[wd*wd];
float *m;
const float sigma2 = (2.5*2.5)*(radius*roi_in->scale/piece->iscale)*(radius*roi_in->scale/piece->iscale);
float weight = 0.0f;
memset(mat, 0, wd*wd*sizeof(float));
m = mat;
for(int l=-rad; l<=rad; l++) for(int k=-rad; k<=rad; k++,m++)
weight += *m = expf(- (l*l + k*k)/(2.f*sigma2));
m = mat;
for(int l=-rad; l<=rad; l++) for(int k=-rad; k<=rad; k++,m++)
*m /= weight;
/* gauss blur the L channel */
#ifdef _OPENMP
#pragma omp parallel for default(none) private(in, out, m) shared(mat, ivoid, ovoid, roi_out, roi_in) schedule(static)
#endif
for(int j=rad; j<roi_out->height-rad; j++)
{
in = ((float *)ivoid) + ch*(j*roi_in->width + rad);
out = ((float *)ovoid) + ch*(j*roi_out->width + rad);
for(int i=rad; i<roi_out->width-rad; i++)
{
for(int c=0; c<3; c++) out[c] = 0.0f;
float sum = 0.0;
m = mat;
for(int l=-rad; l<=rad; l++)
{
float *inrow = in + ch*(l*roi_in->width-rad);
for(int k=-rad; k<=rad; k++,inrow+=ch,m++)
sum += *m * inrow[0];
}
out[0] = sum;
out += ch;
in += ch;
}
}
/* create zonemap preview */
// in = (float *)ivoid;
out = (float *)ovoid;
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(roi_out,out,buffer,g,zonemap) schedule(static)
#endif
for (int k=0; k<roi_out->width*roi_out->height; k++)
{
buffer[k] = _iop_zonesystem_zone_index_from_lightness (out[ch*k]/100.0f, zonemap, size);
}
dt_pthread_mutex_unlock(&g->lock);
}
/* process the image */
in = (float *)ivoid;
out = (float *)ovoid;
const float rzscale = (size-1)/100.0f;
float zonemap_offset[MAX_ZONE_SYSTEM_SIZE]= {-1};
float zonemap_scale[MAX_ZONE_SYSTEM_SIZE]= {-1};
// precompute scale and offset
for (int k=0; k < size-1; k++) zonemap_scale[k] = (zonemap[k+1]-zonemap[k])*(size-1);
for (int k=0; k < size-1; k++) zonemap_offset[k] = 100.0f * ((k+1)*zonemap[k] - k*zonemap[k+1]) ;
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(roi_out, in, out, zonemap_scale,zonemap_offset) schedule(static)
#endif
for (int j=0; j<roi_out->height; j++)
for (int i=0; i<roi_out->width; i++)
{
/* remap lightness into zonemap and apply lightness */
const float *inp = in + ch*(j*roi_out->width+i);
float *outp = out + ch*(j*roi_out->width+i);
const int rz = CLAMPS(inp[0]*rzscale, 0, size-2); // zone index
const float zs = ((rz > 0) ? (zonemap_offset[rz]/inp[0]) : 0) + zonemap_scale[rz];
_mm_stream_ps(outp,_mm_mul_ps(_mm_load_ps(inp),_mm_set1_ps(zs)));
}
_mm_sfence();
if(piece->pipe->mask_display)
dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height);
}
/* A vectorized version of the Voigt function using X86 SSE instructions */
void my_voigt(const float *damping, const float *frequency_offset, float *voigt_value, int N)
{
// coefficients of the rational approximation formula
// to the complementary error function
const __m128 A0 = _mm_set1_ps(122.607931777104326f);
const __m128 A1 = _mm_set1_ps(214.382388694706425f);
const __m128 A2 = _mm_set1_ps(181.928533092181549f);
const __m128 A3 = _mm_set1_ps(93.155580458138441f);
const __m128 A4 = _mm_set1_ps(30.180142196210589f);
const __m128 A5 = _mm_set1_ps(5.912626209773153f);
const __m128 A6 = _mm_set1_ps(0.564189583562615f);
const __m128 B0 = _mm_set1_ps(122.60793177387535f);
const __m128 B1 = _mm_set1_ps(352.730625110963558f);
const __m128 B2 = _mm_set1_ps(457.334478783897737f);
const __m128 B3 = _mm_set1_ps(348.703917719495792f);
const __m128 B4 = _mm_set1_ps(170.354001821091472f);
const __m128 B5 = _mm_set1_ps(53.992906912940207f);
const __m128 B6 = _mm_set1_ps(10.479857114260399f);
__m128 ivsigno;
__m128 V;
__m128 Z1_real;
__m128 Z1_imag;
__m128 Z2_real;
__m128 Z2_imag;
__m128 Z3_real;
__m128 Z3_imag;
__m128 Z4_real;
__m128 Z4_imag;
__m128 Z5_real;
__m128 Z5_imag;
__m128 Z6_real;
__m128 Z6_imag;
__m128 ZZ1_real;
__m128 ZZ1_imag;
__m128 ZZ2_real;
__m128 ZZ2_imag;
__m128 ZZ3_real;
__m128 ZZ3_imag;
__m128 ZZ4_real;
__m128 ZZ4_imag;
__m128 ZZ5_real;
__m128 ZZ5_imag;
__m128 ZZ6_real;
__m128 ZZ6_imag;
__m128 ZZ7_real;
__m128 ZZ7_imag;
__m128 division_factor;
__m128 ZZZ_real;
__m128 damp;
__m128 offs;
__m128 vval;
__m128 one = _mm_set1_ps(1.0f);
__m128 zero = _mm_set1_ps(0.0f);
__m128 mone = _mm_set1_ps(-1.0f);
__m128 half = _mm_set1_ps(-0.5f);
__m128 mask;
float *stmp = (float *) _mm_malloc(4*sizeof(float), 16);
int i;
for(i=0; i<N; i+=VECLEN){
_mm_prefetch((const char *)&damping[i+64], _MM_HINT_T0);
_mm_prefetch((const char *)&frequency_offset[i+64], _MM_HINT_T0);
damp = _mm_load_ps(&damping[i]);
offs = _mm_load_ps(&frequency_offset[i]);
mask = _mm_cmplt_ps(offs, zero);
ivsigno = _mm_add_ps(_mm_and_ps(mask,mone),_mm_andnot_ps(mask,one));
V = _mm_mul_ps(ivsigno, offs);
Z1_real = _mm_add_ps(_mm_mul_ps(A6, damp), A5);
Z1_imag = _mm_mul_ps(A6, V);
Z2_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(Z1_real, damp), _mm_mul_ps(Z1_imag, V)), A4);
Z2_imag = _mm_add_ps(_mm_mul_ps(Z1_real, V), _mm_mul_ps(Z1_imag, damp));
Z3_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(Z2_real, damp), _mm_mul_ps(Z2_imag, V)), A3);
Z3_imag = _mm_add_ps(_mm_mul_ps(Z2_real, V), _mm_mul_ps(Z2_imag, damp));
Z4_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(Z3_real, damp), _mm_mul_ps(Z3_imag, V)), A2);
Z4_imag = _mm_add_ps(_mm_mul_ps(Z3_real, V), _mm_mul_ps(Z3_imag, damp));
Z5_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(Z4_real, damp), _mm_mul_ps(Z4_imag, V)), A1);
Z5_imag = _mm_add_ps(_mm_mul_ps(Z4_real, V), _mm_mul_ps(Z4_imag, damp));
Z6_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(Z5_real, damp), _mm_mul_ps(Z5_imag, V)), A0);
Z6_imag = _mm_add_ps(_mm_mul_ps(Z5_real, V), _mm_mul_ps(Z5_imag, damp));
ZZ1_real = _mm_add_ps(damp, B6);
ZZ1_imag = V;
ZZ2_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(ZZ1_real, damp), _mm_mul_ps(ZZ1_imag, V)), B5);
ZZ2_imag = _mm_add_ps(_mm_mul_ps(ZZ1_real, V), _mm_mul_ps(ZZ1_imag, damp));
ZZ3_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(ZZ2_real, damp), _mm_mul_ps(ZZ2_imag, V)), B4);
ZZ3_imag = _mm_add_ps(_mm_mul_ps(ZZ2_real, V), _mm_mul_ps(ZZ2_imag, damp));
ZZ4_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(ZZ3_real, damp), _mm_mul_ps(ZZ3_imag, V)), B3);
ZZ4_imag = _mm_add_ps(_mm_mul_ps(ZZ3_real, V), _mm_mul_ps(ZZ3_imag, damp));
ZZ5_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(ZZ4_real, damp), _mm_mul_ps(ZZ4_imag, V)), B2);
ZZ5_imag = _mm_add_ps(_mm_mul_ps(ZZ4_real, V), _mm_mul_ps(ZZ4_imag, damp));
ZZ6_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(ZZ5_real, damp), _mm_mul_ps(ZZ5_imag, V)), B1);
ZZ6_imag = _mm_add_ps(_mm_mul_ps(ZZ5_real, V), _mm_mul_ps(ZZ5_imag, damp));
ZZ7_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(ZZ6_real, damp), _mm_mul_ps(ZZ6_imag, V)), B0);
ZZ7_imag = _mm_add_ps(_mm_mul_ps(ZZ6_real, V), _mm_mul_ps(ZZ6_imag, damp));
division_factor = _mm_div_ps(one, _mm_add_ps(_mm_mul_ps(ZZ7_real, ZZ7_real), _mm_mul_ps(ZZ7_imag, ZZ7_imag)));
ZZZ_real = _mm_mul_ps((_mm_add_ps(_mm_mul_ps(Z6_real, ZZ7_real), _mm_mul_ps(Z6_imag, ZZ7_imag))), division_factor);
_mm_stream_ps(&voigt_value[i], ZZZ_real);
}
_mm_free(stmp);
}
void
process (struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid, const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out)
{
const dt_iop_colorout_data_t *const d = (dt_iop_colorout_data_t *)piece->data;
const int ch = piece->colors;
const int gamutcheck = (d->softproof_enabled == DT_SOFTPROOF_GAMUTCHECK);
if(!isnan(d->cmatrix[0]))
{
//fprintf(stderr,"Using cmatrix codepath\n");
// convert to rgb using matrix
#ifdef _OPENMP
#pragma omp parallel for schedule(static) default(none) shared(roi_in,roi_out, ivoid, ovoid)
#endif
for(int j=0; j<roi_out->height; j++)
{
float *in = (float*)ivoid + ch*roi_in->width *j;
float *out = (float*)ovoid + ch*roi_out->width*j;
const __m128 m0 = _mm_set_ps(0.0f,d->cmatrix[6],d->cmatrix[3],d->cmatrix[0]);
const __m128 m1 = _mm_set_ps(0.0f,d->cmatrix[7],d->cmatrix[4],d->cmatrix[1]);
const __m128 m2 = _mm_set_ps(0.0f,d->cmatrix[8],d->cmatrix[5],d->cmatrix[2]);
for(int i=0; i<roi_out->width; i++, in+=ch, out+=ch )
{
const __m128 xyz = dt_Lab_to_XYZ_SSE(_mm_load_ps(in));
const __m128 t = _mm_add_ps(_mm_mul_ps(m0,_mm_shuffle_ps(xyz,xyz,_MM_SHUFFLE(0,0,0,0))),_mm_add_ps(_mm_mul_ps(m1,_mm_shuffle_ps(xyz,xyz,_MM_SHUFFLE(1,1,1,1))),_mm_mul_ps(m2,_mm_shuffle_ps(xyz,xyz,_MM_SHUFFLE(2,2,2,2)))));
_mm_stream_ps(out,t);
}
}
_mm_sfence();
// apply profile
#ifdef _OPENMP
#pragma omp parallel for schedule(static) default(none) shared(roi_in,roi_out, ivoid, ovoid)
#endif
for(int j=0; j<roi_out->height; j++)
{
float *in = (float*)ivoid + ch*roi_in->width *j;
float *out = (float*)ovoid + ch*roi_out->width*j;
for(int i=0; i<roi_out->width; i++, in+=ch, out+=ch )
{
for(int i=0; i<3; i++)
if (d->lut[i][0] >= 0.0f)
{
out[i] = (out[i] < 1.0f) ? lerp_lut(d->lut[i], out[i]) : dt_iop_eval_exp(d->unbounded_coeffs[i], out[i]);
}
}
}
}
else
{
float *in = (float*)ivoid;
float *out = (float*)ovoid;
const int rowsize=roi_out->width * 3;
//fprintf(stderr,"Using xform codepath\n");
#ifdef _OPENMP
#pragma omp parallel for schedule(static) default(none) shared(out, roi_out, in)
#endif
for (int k=0; k<roi_out->height; k++)
{
float Lab[rowsize];
float rgb[rowsize];
const int m=(k*(roi_out->width*ch));
for (int l=0; l<roi_out->width; l++)
{
int li=3*l,ii=ch*l;
Lab[li+0] = in[m+ii+0];
Lab[li+1] = in[m+ii+1];
Lab[li+2] = in[m+ii+2];
}
cmsDoTransform (d->xform, Lab, rgb, roi_out->width);
for (int l=0; l<roi_out->width; l++)
{
int oi=ch*l, ri=3*l;
if(gamutcheck && (rgb[ri+0] < 0.0f || rgb[ri+1] < 0.0f || rgb[ri+2] < 0.0f))
{
out[m+oi+0] = 0.0f;
out[m+oi+1] = 1.0f;
out[m+oi+2] = 1.0f;
}
else
{
out[m+oi+0] = rgb[ri+0];
out[m+oi+1] = rgb[ri+1];
out[m+oi+2] = rgb[ri+2];
}
}
}
}
if(piece->pipe->mask_display)
dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height);
}
void process (struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid, const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out)
{
const int filters = dt_image_flipped_filter(&piece->pipe->image);
dt_iop_temperature_data_t *d = (dt_iop_temperature_data_t *)piece->data;
if(!dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) && filters && piece->pipe->image.bpp != 4)
{
const float coeffsi[3] = {d->coeffs[0]/65535.0f, d->coeffs[1]/65535.0f, d->coeffs[2]/65535.0f};
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid, d) schedule(static)
#endif
for(int j=0; j<roi_out->height; j++)
{
int i=0;
const uint16_t *in = ((uint16_t *)ivoid) + j*roi_out->width;
float *out = ((float*)ovoid) + j*roi_out->width;
// process unaligned pixels
for ( ; i < ((4-(j*roi_out->width & 3)) & 3) ; i++,out++,in++)
*out = *in * coeffsi[FC(j+roi_out->y, i+roi_out->x, filters)];
const __m128 coeffs = _mm_set_ps(coeffsi[FC(j+roi_out->y, roi_out->x+i+3, filters)],
coeffsi[FC(j+roi_out->y, roi_out->x+i+2, filters)],
coeffsi[FC(j+roi_out->y, roi_out->x+i+1, filters)],
coeffsi[FC(j+roi_out->y, roi_out->x+i , filters)]);
// process aligned pixels with SSE
for( ; i < roi_out->width - 3 ; i+=4,out+=4,in+=4)
{
_mm_stream_ps(out,_mm_mul_ps(coeffs,_mm_set_ps(in[3],in[2],in[1],in[0])));
}
// process the rest
for( ; i<roi_out->width; i++,out++,in++)
*out = *in * coeffsi[FC(j+roi_out->y, i+roi_out->x, filters)];
}
_mm_sfence();
}
else if(!dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) && filters && piece->pipe->image.bpp == 4)
{
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid, d) schedule(static)
#endif
for(int j=0; j<roi_out->height; j++)
{
const float *in = ((float *)ivoid) + j*roi_out->width;
float *out = ((float*)ovoid) + j*roi_out->width;
for(int i=0; i<roi_out->width; i++,out++,in++)
*out = *in * d->coeffs[FC(j+roi_out->x, i+roi_out->y, filters)];
}
}
else
{
const int ch = piece->colors;
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid, d) schedule(static)
#endif
for(int k=0; k<roi_out->height; k++)
{
const float *in = ((float*)ivoid) + ch*k*roi_out->width;
float *out = ((float*)ovoid) + ch*k*roi_out->width;
for (int j=0; j<roi_out->width; j++,in+=ch,out+=ch)
for(int c=0; c<3; c++) out[c] = in[c]*d->coeffs[c];
}
}
for(int k=0; k<3; k++)
piece->pipe->processed_maximum[k] = d->coeffs[k] * piece->pipe->processed_maximum[k];
}
static gboolean draw(GtkWidget *widget, cairo_t *cr, dt_iop_module_t *self)
{
if(darktable.gui->reset) return FALSE;
if(self->picked_color_max[0] < 0.0f) return FALSE;
if(self->request_color_pick == DT_REQUEST_COLORPICK_OFF) return FALSE;
dt_iop_invert_gui_data_t *g = (dt_iop_invert_gui_data_t *)self->gui_data;
dt_iop_invert_params_t *p = (dt_iop_invert_params_t *)self->params;
if(fabsf(p->color[0] - self->picked_color[0]) < 0.0001f
&& fabsf(p->color[1] - self->picked_color[1]) < 0.0001f
&& fabsf(p->color[2] - self->picked_color[2]) < 0.0001f)
{
// interrupt infinite loops
return FALSE;
}
p->color[0] = self->picked_color[0];
p->color[1] = self->picked_color[1];
p->color[2] = self->picked_color[2];
GdkRGBA color = (GdkRGBA){.red = p->color[0], .green = p->color[1], .blue = p->color[2], .alpha = 1.0 };
gtk_color_chooser_set_rgba(GTK_COLOR_CHOOSER(g->colorpicker), &color);
dt_dev_add_history_item(darktable.develop, self, TRUE);
return FALSE;
}
static void colorpicker_callback(GtkColorButton *widget, dt_iop_module_t *self)
{
if(self->dt->gui->reset) return;
dt_iop_invert_gui_data_t *g = (dt_iop_invert_gui_data_t *)self->gui_data;
dt_iop_invert_params_t *p = (dt_iop_invert_params_t *)self->params;
// turn off the other color picker so that this tool actually works ...
gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(g->picker), FALSE);
GdkRGBA c;
gtk_color_chooser_get_rgba(GTK_COLOR_CHOOSER(widget), &c);
p->color[0] = c.red;
p->color[1] = c.green;
p->color[2] = c.blue;
dt_dev_add_history_item(darktable.develop, self, TRUE);
}
static int FC(const int row, const int col, const unsigned int filters)
{
return filters >> (((row << 1 & 14) + (col & 1)) << 1) & 3;
}
static uint8_t FCxtrans(const int row, const int col, const dt_iop_roi_t *const roi,
uint8_t (*const xtrans)[6])
{
return xtrans[(row + roi->y) % 6][(col + roi->x) % 6];
}
void process(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid,
const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out)
{
dt_iop_invert_data_t *d = (dt_iop_invert_data_t *)piece->data;
const float *const m = piece->pipe->processed_maximum;
const float film_rgb[3] = { d->color[0], d->color[1], d->color[2] };
const float film_rgb_f[3] = { d->color[0] * m[0], d->color[1] * m[1], d->color[2] * m[2] };
// FIXME: it could be wise to make this a NOP when picking colors. not sure about that though.
// if(self->request_color_pick){
// do nothing
// }
const int filters = dt_image_filter(&piece->pipe->image);
uint8_t (*const xtrans)[6] = self->dev->image_storage.xtrans;
if(!dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) && (filters == 9u))
{ // xtrans float mosaiced
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid) schedule(static)
#endif
for(int j = 0; j < roi_out->height; j++)
{
const float *in = ((float *)ivoid) + (size_t)j * roi_out->width;
float *out = ((float *)ovoid) + (size_t)j * roi_out->width;
for(int i = 0; i < roi_out->width; i++, out++, in++)
*out = CLAMP(film_rgb_f[FCxtrans(j, i, roi_out, xtrans)] - *in, 0.0f, 1.0f);
}
for(int k = 0; k < 3; k++) piece->pipe->processed_maximum[k] = 1.0f;
}
else if(!dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) && filters)
{ // bayer float mosaiced
const __m128 val_min = _mm_setzero_ps();
const __m128 val_max = _mm_set1_ps(1.0f);
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid) schedule(static)
#endif
for(int j = 0; j < roi_out->height; j++)
{
const float *in = ((float *)ivoid) + (size_t)j * roi_out->width;
float *out = ((float *)ovoid) + (size_t)j * roi_out->width;
int i = 0;
int alignment = ((4 - (j * roi_out->width & (4 - 1))) & (4 - 1));
// process unaligned pixels
for(; i < alignment; i++, out++, in++)
*out = CLAMP(film_rgb_f[FC(j + roi_out->y, i + roi_out->x, filters)] - *in, 0.0f, 1.0f);
const __m128 film = _mm_set_ps(film_rgb_f[FC(j + roi_out->y, roi_out->x + i + 3, filters)],
film_rgb_f[FC(j + roi_out->y, roi_out->x + i + 2, filters)],
film_rgb_f[FC(j + roi_out->y, roi_out->x + i + 1, filters)],
film_rgb_f[FC(j + roi_out->y, roi_out->x + i, filters)]);
// process aligned pixels with SSE
for(; i < roi_out->width - (4 - 1); i += 4, in += 4, out += 4)
{
const __m128 input = _mm_load_ps(in);
const __m128 subtracted = _mm_sub_ps(film, input);
_mm_stream_ps(out, _mm_max_ps(_mm_min_ps(subtracted, val_max), val_min));
}
// process the rest
for(; i < roi_out->width; i++, out++, in++)
*out = CLAMP(film_rgb_f[FC(j + roi_out->y, i + roi_out->x, filters)] - *in, 0.0f, 1.0f);
}
_mm_sfence();
for(int k = 0; k < 3; k++) piece->pipe->processed_maximum[k] = 1.0f;
}
else
{ // non-mosaiced
const int ch = piece->colors;
const __m128 film = _mm_set_ps(1.0f, film_rgb[2], film_rgb[1], film_rgb[0]);
#ifdef _OPENMP
#pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid) schedule(static)
#endif
for(int k = 0; k < roi_out->height; k++)
{
const float *in = ((float *)ivoid) + (size_t)ch * k * roi_out->width;
float *out = ((float *)ovoid) + (size_t)ch * k * roi_out->width;
for(int j = 0; j < roi_out->width; j++, in += ch, out += ch)
{
const __m128 input = _mm_load_ps(in);
const __m128 subtracted = _mm_sub_ps(film, input);
_mm_stream_ps(out, subtracted);
}
}
_mm_sfence();
if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height);
}
}
