void process_sse2(dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, const void *const ivoid, void *const ovoid, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out) { const dt_iop_rawprepare_data_t *const d = (dt_iop_rawprepare_data_t *)piece->data; // fprintf(stderr, "roi in %d %d %d %d\n", roi_in->x, roi_in->y, roi_in->width, roi_in->height); // fprintf(stderr, "roi out %d %d %d %d\n", roi_out->x, roi_out->y, roi_out->width, roi_out->height); const float scale = roi_in->scale / piece->iscale; const int csx = (int)roundf((float)d->x * scale), csy = (int)roundf((float)d->y * scale); if(!dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) && piece->pipe->filters) { // raw mosaic #ifdef _OPENMP #pragma omp parallel for default(none) schedule(static) #endif for(int j = 0; j < roi_out->height; j++) { const uint16_t *in = ((uint16_t *)ivoid) + ((size_t)roi_in->width * (j + csy) + csx); float *out = ((float *)ovoid) + (size_t)roi_out->width * j; int i = 0; // FIXME: figure alignment! !!! replace with for !!! while((!dt_is_aligned(in, 16) || !dt_is_aligned(out, 16)) && (i < roi_out->width)) { const int id = BL(roi_out, d, j, i); *out = (((float)(*in)) - d->sub[id]) / d->div[id]; i++; in++; out++; } const __m128 sub = _mm_set_ps(d->sub[BL(roi_out, d, j, i + 3)], d->sub[BL(roi_out, d, j, i + 2)], d->sub[BL(roi_out, d, j, i + 1)], d->sub[BL(roi_out, d, j, i)]); const __m128 div = _mm_set_ps(d->div[BL(roi_out, d, j, i + 3)], d->div[BL(roi_out, d, j, i + 2)], d->div[BL(roi_out, d, j, i + 1)], d->div[BL(roi_out, d, j, i)]); // process aligned pixels with SSE for(; i < roi_out->width - (8 - 1); i += 8, in += 8) { const __m128i input = _mm_load_si128((__m128i *)in); __m128i ilo = _mm_unpacklo_epi16(input, _mm_set1_epi16(0)); __m128i ihi = _mm_unpackhi_epi16(input, _mm_set1_epi16(0)); __m128 flo = _mm_cvtepi32_ps(ilo); __m128 fhi = _mm_cvtepi32_ps(ihi); flo = _mm_div_ps(_mm_sub_ps(flo, sub), div); fhi = _mm_div_ps(_mm_sub_ps(fhi, sub), div); _mm_stream_ps(out, flo); out += 4; _mm_stream_ps(out, fhi); out += 4; } // process the rest for(; i < roi_out->width; i++, in++, out++) { const int id = BL(roi_out, d, j, i); *out = MAX(0.0f, ((float)(*in)) - d->sub[id]) / d->div[id]; } } piece->pipe->filters = dt_rawspeed_crop_dcraw_filters(piece->pipe->filters, csx, csy); adjust_xtrans_filters(piece->pipe->xtrans, csx, csy); } else { // pre-downsampled buffer that needs black/white scaling const __m128 sub = _mm_load_ps(d->sub), div = _mm_load_ps(d->div); #ifdef _OPENMP #pragma omp parallel for default(none) schedule(static) #endif for(int j = 0; j < roi_out->height; j++) { const float *in = ((float *)ivoid) + (size_t)4 * (roi_in->width * (j + csy) + csx); float *out = ((float *)ovoid) + (size_t)4 * roi_out->width * j; // process aligned pixels with SSE for(int i = 0; i < roi_out->width; i++, in += 4, out += 4) { const __m128 input = _mm_load_ps(in); const __m128 scaled = _mm_div_ps(_mm_sub_ps(input, sub), div); _mm_stream_ps(out, scaled); } } } _mm_sfence(); }
void process(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid, const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out) { const dt_iop_colorout_data_t *const d = (dt_iop_colorout_data_t *)piece->data; const int ch = piece->colors; const int gamutcheck = (d->softproof_enabled == DT_SOFTPROOF_GAMUTCHECK); if(!isnan(d->cmatrix[0])) { // fprintf(stderr,"Using cmatrix codepath\n"); // convert to rgb using matrix #ifdef _OPENMP #pragma omp parallel for schedule(static) default(none) shared(roi_in, roi_out, ivoid, ovoid) #endif for(int j = 0; j < roi_out->height; j++) { float *in = (float *)ivoid + (size_t)ch * roi_in->width * j; float *out = (float *)ovoid + (size_t)ch * roi_out->width * j; const __m128 m0 = _mm_set_ps(0.0f, d->cmatrix[6], d->cmatrix[3], d->cmatrix[0]); const __m128 m1 = _mm_set_ps(0.0f, d->cmatrix[7], d->cmatrix[4], d->cmatrix[1]); const __m128 m2 = _mm_set_ps(0.0f, d->cmatrix[8], d->cmatrix[5], d->cmatrix[2]); for(int i = 0; i < roi_out->width; i++, in += ch, out += ch) { const __m128 xyz = dt_Lab_to_XYZ_SSE(_mm_load_ps(in)); const __m128 t = _mm_add_ps(_mm_mul_ps(m0, _mm_shuffle_ps(xyz, xyz, _MM_SHUFFLE(0, 0, 0, 0))), _mm_add_ps(_mm_mul_ps(m1, _mm_shuffle_ps(xyz, xyz, _MM_SHUFFLE(1, 1, 1, 1))), _mm_mul_ps(m2, _mm_shuffle_ps(xyz, xyz, _MM_SHUFFLE(2, 2, 2, 2))))); _mm_stream_ps(out, t); } } _mm_sfence(); // apply profile #ifdef _OPENMP #pragma omp parallel for schedule(static) default(none) shared(roi_in, roi_out, ivoid, ovoid) #endif for(int j = 0; j < roi_out->height; j++) { float *in = (float *)ivoid + (size_t)ch * roi_in->width * j; float *out = (float *)ovoid + (size_t)ch * roi_out->width * j; for(int i = 0; i < roi_out->width; i++, in += ch, out += ch) { for(int i = 0; i < 3; i++) if(d->lut[i][0] >= 0.0f) { out[i] = (out[i] < 1.0f) ? lerp_lut(d->lut[i], out[i]) : dt_iop_eval_exp(d->unbounded_coeffs[i], out[i]); } } } } else { // fprintf(stderr,"Using xform codepath\n"); const __m128 outofgamutpixel = _mm_set_ps(0.0f, 1.0f, 1.0f, 0.0f); #ifdef _OPENMP #pragma omp parallel for schedule(static) default(none) shared(ivoid, ovoid, roi_out) #endif for(int k = 0; k < roi_out->height; k++) { const float *in = ((float *)ivoid) + (size_t)ch * k * roi_out->width; float *out = ((float *)ovoid) + (size_t)ch * k * roi_out->width; if(!gamutcheck) { cmsDoTransform(d->xform, in, out, roi_out->width); } else { void *rgb = dt_alloc_align(16, 4 * sizeof(float) * roi_out->width); cmsDoTransform(d->xform, in, rgb, roi_out->width); float *rgbptr = (float *)rgb; for(int j = 0; j < roi_out->width; j++, rgbptr += 4, out += 4) { const __m128 pixel = _mm_load_ps(rgbptr); __m128 ingamut = _mm_cmplt_ps(pixel, _mm_set_ps(-FLT_MAX, 0.0f, 0.0f, 0.0f)); ingamut = _mm_or_ps(_mm_unpacklo_ps(ingamut, ingamut), _mm_unpackhi_ps(ingamut, ingamut)); ingamut = _mm_or_ps(_mm_unpacklo_ps(ingamut, ingamut), _mm_unpackhi_ps(ingamut, ingamut)); const __m128 result = _mm_or_ps(_mm_and_ps(ingamut, outofgamutpixel), _mm_andnot_ps(ingamut, pixel)); _mm_stream_ps(out, result); } dt_free_align(rgb); } } _mm_sfence(); } if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height); }
int main() { float *arr = get_arr(); // [4, 3, 2, 1] float *uarr = get_uarr(); // [5, 4, 3, 2] float *arr2 = get_arr2(); // [4, 3, 2, 1] float *uarr2 = get_uarr2(); // [5, 4, 3, 2] __m128 a = get_a(); // [8, 6, 4, 2] __m128 b = get_b(); // [1, 2, 3, 4] // Check that test data is like expected. Assert(((uintptr_t)arr & 0xF) == 0); // arr must be aligned by 16. Assert(((uintptr_t)uarr & 0xF) != 0); // uarr must be unaligned. Assert(((uintptr_t)arr2 & 0xF) == 0); // arr must be aligned by 16. Assert(((uintptr_t)uarr2 & 0xF) != 0); // uarr must be unaligned. // Test that aeq itself works and does not trivially return true on everything. Assert(aeq_("",_mm_load_ps(arr), 4.f, 3.f, 2.f, 0.f, false) == false); #ifdef TEST_M64 Assert(aeq64(u64castm64(0x22446688AACCEEFFULL), 0xABABABABABABABABULL, false) == false); #endif // SSE1 Load instructions: aeq(_mm_load_ps(arr), 4.f, 3.f, 2.f, 1.f); // 4-wide load from aligned address. aeq(_mm_load_ps1(uarr), 2.f, 2.f, 2.f, 2.f); // Load scalar from unaligned address and populate 4-wide. aeq(_mm_load_ss(uarr), 0.f, 0.f, 0.f, 2.f); // Load scalar from unaligned address to lowest, and zero all highest. aeq(_mm_load1_ps(uarr), 2.f, 2.f, 2.f, 2.f); // _mm_load1_ps == _mm_load_ps1 aeq(_mm_loadh_pi(a, (__m64*)uarr), 3.f, 2.f, 4.f, 2.f); // Load two highest addresses, preserve two lowest. aeq(_mm_loadl_pi(a, (__m64*)uarr), 8.f, 6.f, 3.f, 2.f); // Load two lowest addresses, preserve two highest. aeq(_mm_loadr_ps(arr), 1.f, 2.f, 3.f, 4.f); // 4-wide load from an aligned address, but reverse order. aeq(_mm_loadu_ps(uarr), 5.f, 4.f, 3.f, 2.f); // 4-wide load from an unaligned address. // SSE1 Set instructions: aeq(_mm_set_ps(uarr[3], 2.f, 3.f, 4.f), 5.f, 2.f, 3.f, 4.f); // 4-wide set by specifying four immediate or memory operands. aeq(_mm_set_ps1(uarr[3]), 5.f, 5.f, 5.f, 5.f); // 4-wide set by specifying one scalar that is expanded. aeq(_mm_set_ss(uarr[3]), 0.f, 0.f, 0.f, 5.f); // Set scalar at lowest index, zero all higher. aeq(_mm_set1_ps(uarr[3]), 5.f, 5.f, 5.f, 5.f); // _mm_set1_ps == _mm_set_ps1 aeq(_mm_setr_ps(uarr[3], 2.f, 3.f, 4.f), 4.f, 3.f, 2.f, 5.f); // 4-wide set by specifying four immediate or memory operands, but reverse order. aeq(_mm_setzero_ps(), 0.f, 0.f, 0.f, 0.f); // Returns a new zero register. // SSE1 Move instructions: aeq(_mm_move_ss(a, b), 8.f, 6.f, 4.f, 4.f); // Copy three highest elements from a, and lowest from b. aeq(_mm_movehl_ps(a, b), 8.f, 6.f, 1.f, 2.f); // Copy two highest elements from a, and take two highest from b and place them to the two lowest in output. aeq(_mm_movelh_ps(a, b), 3.f, 4.f, 4.f, 2.f); // Copy two lowest elements from a, and take two lowest from b and place them to the two highest in output. // SSE1 Store instructions: #ifdef TEST_M64 /*M64*/*(uint64_t*)uarr = 0xCDCDCDCDCDCDCDCDULL; _mm_maskmove_si64(u64castm64(0x00EEDDCCBBAA9988ULL), u64castm64(0x0080FF7F01FEFF40ULL), (char*)uarr); Assert(*(uint64_t*)uarr == 0xCDEEDDCDCDAA99CDULL); // _mm_maskmove_si64: Conditionally store bytes of a 64-bit value. /*M64*/*(uint64_t*)uarr = 0xABABABABABABABABULL; _m_maskmovq(u64castm64(0x00EEDDCCBBAA9988ULL), u64castm64(0x0080FF7F01FEFF40ULL), (char*)uarr); Assert(*(uint64_t*)uarr == 0xABEEDDABABAA99ABULL); // _m_maskmovq is an alias to _mm_maskmove_si64. #endif _mm_store_ps(arr2, a); aeq(_mm_load_ps(arr2), 8.f, 6.f, 4.f, 2.f); // _mm_store_ps: 4-wide store to aligned memory address. _mm_store_ps1(arr2, a); aeq(_mm_load_ps(arr2), 2.f, 2.f, 2.f, 2.f); // _mm_store_ps1: Store lowest scalar to aligned address, duplicating the element 4 times. _mm_storeu_ps(uarr2, _mm_set1_ps(100.f)); _mm_store_ss(uarr2, b); aeq(_mm_loadu_ps(uarr2), 100.f, 100.f, 100.f, 4.f); // _mm_store_ss: Store lowest scalar to unaligned address. Don't adjust higher addresses in memory. _mm_store_ps(arr2, _mm_set1_ps(100.f)); _mm_store1_ps(arr2, a); aeq(_mm_load_ps(arr2), 2.f, 2.f, 2.f, 2.f); // _mm_store1_ps == _mm_store_ps1 _mm_storeu_ps(uarr2, _mm_set1_ps(100.f)); _mm_storeh_pi((__m64*)uarr2, a); aeq(_mm_loadu_ps(uarr2), 100.f, 100.f, 8.f, 6.f); // _mm_storeh_pi: Store two highest elements to memory. _mm_storeu_ps(uarr2, _mm_set1_ps(100.f)); _mm_storel_pi((__m64*)uarr2, a); aeq(_mm_loadu_ps(uarr2), 100.f, 100.f, 4.f, 2.f); // _mm_storel_pi: Store two lowest elements to memory. _mm_storer_ps(arr2, a); aeq(_mm_load_ps(arr2), 2.f, 4.f, 6.f, 8.f); // _mm_storer_ps: 4-wide store to aligned memory address, but reverse the elements on output. _mm_storeu_ps(uarr2, a); aeq(_mm_loadu_ps(uarr2), 8.f, 6.f, 4.f, 2.f); // _mm_storeu_ps: 4-wide store to unaligned memory address. #ifdef TEST_M64 /*M64*/_mm_stream_pi((__m64*)uarr, u64castm64(0x0080FF7F01FEFF40ULL)); Assert(*(uint64_t*)uarr == 0x0080FF7F01FEFF40ULL); // _mm_stream_pi: 2-wide store, but with a non-temporal memory cache hint. #endif _mm_store_ps(arr2, _mm_set1_ps(100.f)); _mm_stream_ps(arr2, a); aeq(_mm_load_ps(arr2), 8.f, 6.f, 4.f, 2.f); // _mm_stream_ps: 4-wide store, but with a non-temporal memory cache hint. // SSE1 Arithmetic instructions: aeq(_mm_add_ps(a, b), 9.f, 8.f, 7.f, 6.f); // 4-wide add. aeq(_mm_add_ss(a, b), 8.f, 6.f, 4.f, 6.f); // Add lowest element, preserve three highest unchanged from a. aeq(_mm_div_ps(a, _mm_set_ps(2.f, 3.f, 8.f, 2.f)), 4.f, 2.f, 0.5f, 1.f); // 4-wide div. aeq(_mm_div_ss(a, _mm_set_ps(2.f, 3.f, 8.f, 8.f)), 8.f, 6.f, 4.f, 0.25f); // Div lowest element, preserve three highest unchanged from a. aeq(_mm_mul_ps(a, b), 8.f, 12.f, 12.f, 8.f); // 4-wide mul. aeq(_mm_mul_ss(a, b), 8.f, 6.f, 4.f, 8.f); // Mul lowest element, preserve three highest unchanged from a. #ifdef TEST_M64 __m64 m1 = get_m1(); /*M64*/aeq64(_mm_mulhi_pu16(m1, u64castm64(0x22446688AACCEEFFULL)), 0x002233440B4C33CFULL); // Multiply u16 channels, and store high parts. /*M64*/aeq64( _m_pmulhuw(m1, u64castm64(0x22446688AACCEEFFULL)), 0x002233440B4C33CFULL); // _m_pmulhuw is an alias to _mm_mulhi_pu16. __m64 m2 = get_m2(); /*M64*/aeq64(_mm_sad_pu8(m1, m2), 0x368ULL); // Compute abs. differences of u8 channels, and sum those up to a single 16-bit scalar. /*M64*/aeq64( _m_psadbw(m1, m2), 0x368ULL); // _m_psadbw is an alias to _mm_sad_pu8. #endif aeq(_mm_sub_ps(a, b), 7.f, 4.f, 1.f, -2.f); // 4-wide sub. aeq(_mm_sub_ss(a, b), 8.f, 6.f, 4.f, -2.f); // Sub lowest element, preserve three highest unchanged from a. // SSE1 Elementary Math functions: #ifndef __EMSCRIPTEN__ // TODO: Enable support for this to pass. aeq(_mm_rcp_ps(a), 0.124969f, 0.166626f, 0.249939f, 0.499878f); // Compute 4-wide 1/x. aeq(_mm_rcp_ss(a), 8.f, 6.f, 4.f, 0.499878f); // Compute 1/x of lowest element, pass higher elements unchanged. aeq(_mm_rsqrt_ps(a), 0.353455f, 0.408203f, 0.499878f, 0.706909f); // Compute 4-wide 1/sqrt(x). aeq(_mm_rsqrt_ss(a), 8.f, 6.f, 4.f, 0.706909f); // Compute 1/sqrt(x) of lowest element, pass higher elements unchanged. #endif aeq(_mm_sqrt_ps(a), 2.82843f, 2.44949f, 2.f, 1.41421f); // Compute 4-wide sqrt(x). aeq(_mm_sqrt_ss(a), 8.f, 6.f, 4.f, 1.41421f); // Compute sqrt(x) of lowest element, pass higher elements unchanged. __m128 i1 = get_i1(); __m128 i2 = get_i2(); // SSE1 Logical instructions: #ifndef __EMSCRIPTEN__ // TODO: The polyfill currently does NaN canonicalization and breaks these. aeqi(_mm_and_ps(i1, i2), 0x83200100, 0x0fecc988, 0x80244021, 0x13458a88); // 4-wide binary AND aeqi(_mm_andnot_ps(i1, i2), 0x388a9888, 0xf0021444, 0x7000289c, 0x00121046); // 4-wide binary (!i1) & i2 aeqi(_mm_or_ps(i1, i2), 0xbfefdba9, 0xffefdfed, 0xf7656bbd, 0xffffdbef); // 4-wide binary OR aeqi(_mm_xor_ps(i1, i2), 0x3ccfdaa9, 0xf0031665, 0x77412b9c, 0xecba5167); // 4-wide binary XOR #endif // SSE1 Compare instructions: // a = [8, 6, 4, 2], b = [1, 2, 3, 4] aeqi(_mm_cmpeq_ps(a, _mm_set_ps(8.f, 0.f, 4.f, 0.f)), 0xFFFFFFFF, 0, 0xFFFFFFFF, 0); // 4-wide cmp == aeqi(_mm_cmpeq_ss(a, _mm_set_ps(8.f, 0.f, 4.f, 2.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0xFFFFFFFF); // scalar cmp ==, pass three highest unchanged. aeqi(_mm_cmpge_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0xFFFFFFFF, 0, 0xFFFFFFFF, 0); // 4-wide cmp >= aeqi(_mm_cmpge_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 0.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0xFFFFFFFF); // scalar cmp >=, pass three highest unchanged. aeqi(_mm_cmpgt_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0, 0, 0xFFFFFFFF, 0); // 4-wide cmp > aeqi(_mm_cmpgt_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 2.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0); // scalar cmp >, pass three highest unchanged. aeqi(_mm_cmple_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0xFFFFFFFF, 0xFFFFFFFF, 0, 0xFFFFFFFF); // 4-wide cmp <= aeqi(_mm_cmple_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 0.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0); // scalar cmp <=, pass three highest unchanged. aeqi(_mm_cmplt_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0, 0xFFFFFFFF, 0, 0xFFFFFFFF); // 4-wide cmp < aeqi(_mm_cmplt_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 2.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0); // scalar cmp <, pass three highest unchanged. aeqi(_mm_cmpneq_ps(a, _mm_set_ps(8.f, 0.f, 4.f, 0.f)), 0, 0xFFFFFFFF, 0, 0xFFFFFFFF); // 4-wide cmp != aeqi(_mm_cmpneq_ss(a, _mm_set_ps(8.f, 0.f, 4.f, 2.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0); // scalar cmp !=, pass three highest unchanged. aeqi(_mm_cmpnge_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0, 0xFFFFFFFF, 0, 0xFFFFFFFF); // 4-wide cmp not >= aeqi(_mm_cmpnge_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 0.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0); // scalar cmp not >=, pass three highest unchanged. aeqi(_mm_cmpngt_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0xFFFFFFFF, 0xFFFFFFFF, 0, 0xFFFFFFFF); // 4-wide cmp not > aeqi(_mm_cmpngt_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 2.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0xFFFFFFFF); // scalar cmp not >, pass three highest unchanged. aeqi(_mm_cmpnle_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0, 0, 0xFFFFFFFF, 0); // 4-wide cmp not <= aeqi(_mm_cmpnle_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 0.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0xFFFFFFFF); // scalar cmp not <=, pass three highest unchanged. aeqi(_mm_cmpnlt_ps(a, _mm_set_ps(8.f, 7.f, 3.f, 5.f)), 0xFFFFFFFF, 0, 0xFFFFFFFF, 0); // 4-wide cmp not < aeqi(_mm_cmpnlt_ss(a, _mm_set_ps(8.f, 7.f, 3.f, 2.f)), fcastu(8.f), fcastu(6.f), fcastu(4.f), 0xFFFFFFFF); // scalar cmp not <, pass three highest unchanged. __m128 nan1 = get_nan1(); // [NAN, 0, 0, NAN] __m128 nan2 = get_nan2(); // [NAN, NAN, 0, 0] aeqi(_mm_cmpord_ps(nan1, nan2), 0, 0, 0xFFFFFFFF, 0); // 4-wide test if both operands are not nan. aeqi(_mm_cmpord_ss(nan1, nan2), fcastu(NAN), 0, 0, 0); // scalar test if both operands are not nan, pass three highest unchanged. // Intel Intrinsics Guide documentation is wrong on _mm_cmpunord_ps and _mm_cmpunord_ss. MSDN is right: http://msdn.microsoft.com/en-us/library/khy6fk1t(v=vs.90).aspx aeqi(_mm_cmpunord_ps(nan1, nan2), 0xFFFFFFFF, 0xFFFFFFFF, 0, 0xFFFFFFFF); // 4-wide test if one of the operands is nan. #ifndef __EMSCRIPTEN__ // TODO: The polyfill currently does NaN canonicalization and breaks these. aeqi(_mm_cmpunord_ss(nan1, nan2), fcastu(NAN), 0, 0, 0xFFFFFFFF); // scalar test if one of the operands is nan, pass three highest unchanged. #endif Assert(_mm_comieq_ss(a, b) == 0); Assert(_mm_comieq_ss(a, a) == 1); // Scalar cmp == of lowest element, return int. Assert(_mm_comige_ss(a, b) == 0); Assert(_mm_comige_ss(a, a) == 1); // Scalar cmp >= of lowest element, return int. Assert(_mm_comigt_ss(b, a) == 1); Assert(_mm_comigt_ss(a, a) == 0); // Scalar cmp > of lowest element, return int. Assert(_mm_comile_ss(b, a) == 0); Assert(_mm_comile_ss(a, a) == 1); // Scalar cmp <= of lowest element, return int. Assert(_mm_comilt_ss(a, b) == 1); Assert(_mm_comilt_ss(a, a) == 0); // Scalar cmp < of lowest element, return int. Assert(_mm_comineq_ss(a, b) == 1); Assert(_mm_comineq_ss(a, a) == 0); // Scalar cmp != of lowest element, return int. // The ucomi versions are identical to comi, except that ucomi signal a FP exception only if one of the input operands is a SNaN, whereas the comi versions signal a FP // exception when one of the input operands is either a QNaN or a SNaN. #ifndef __EMSCRIPTEN__ // TODO: Fix ucomi support in SSE to treat NaNs properly. Assert(_mm_ucomieq_ss(a, b) == 0); Assert(_mm_ucomieq_ss(a, a) == 1); Assert(_mm_ucomieq_ss(a, nan1) == 1); #endif Assert(_mm_ucomige_ss(a, b) == 0); Assert(_mm_ucomige_ss(a, a) == 1); Assert(_mm_ucomige_ss(a, nan1) == 0); Assert(_mm_ucomigt_ss(b, a) == 1); Assert(_mm_ucomigt_ss(a, a) == 0); Assert(_mm_ucomigt_ss(a, nan1) == 0); Assert(_mm_ucomile_ss(b, a) == 0); Assert(_mm_ucomile_ss(a, a) == 1); Assert(_mm_ucomile_ss(a, nan1) == 1); Assert(_mm_ucomilt_ss(a, b) == 1); Assert(_mm_ucomilt_ss(a, a) == 0); Assert(_mm_ucomilt_ss(a, nan1) == 1); #ifndef __EMSCRIPTEN__ // TODO: Fix ucomi support in SSE to treat NaNs properly. Assert(_mm_ucomineq_ss(a, b) == 1); Assert(_mm_ucomineq_ss(a, a) == 0); Assert(_mm_ucomineq_ss(a, nan1) == 0); #endif // SSE1 Convert instructions: __m128 c = get_c(); // [1.5, 2.5, 3.5, 4.5] __m128 e = get_e(); // [INF, -INF, 2.5, 3.5] __m128 f = get_f(); // [-1.5, 1.5, -2.5, -9223372036854775808] #ifdef TEST_M64 /*M64*/aeq(_mm_cvt_pi2ps(a, m2), 8.f, 6.f, -19088744.f, 1985229312.f); // 2-way int32 to float conversion to two lowest channels of m128. /*M64*/aeq64(_mm_cvt_ps2pi(c), 0x400000004ULL); // 2-way two lowest floats from m128 to integer, return as m64. #endif aeq(_mm_cvtsi32_ss(c, -16777215), 1.5f, 2.5f, 3.5f, -16777215.f); // Convert int to float, store in lowest channel of m128. aeq( _mm_cvt_si2ss(c, -16777215), 1.5f, 2.5f, 3.5f, -16777215.f); // _mm_cvt_si2ss is an alias to _mm_cvtsi32_ss. #ifndef __EMSCRIPTEN__ // TODO: Fix banker's rounding in cvt functions. Assert(_mm_cvtss_si32(c) == 4); Assert(_mm_cvtss_si32(e) == 4); // Convert lowest channel of m128 from float to int. Assert( _mm_cvt_ss2si(c) == 4); Assert( _mm_cvt_ss2si(e) == 4); // _mm_cvt_ss2si is an alias to _mm_cvtss_si32. #endif #ifdef TEST_M64 /*M64*/aeq(_mm_cvtpi16_ps(m1), 255.f , -32767.f, 4336.f, 14207.f); // 4-way convert int16s to floats, return in a m128. /*M64*/aeq(_mm_cvtpi32_ps(a, m1), 8.f, 6.f, 16744449.f, 284178304.f); // 2-way convert int32s to floats, return in two lowest channels of m128, pass two highest unchanged. /*M64*/aeq(_mm_cvtpi32x2_ps(m1, m2), -19088744.f, 1985229312.f, 16744449.f, 284178304.f); // 4-way convert int32s from two different m64s to float. /*M64*/aeq(_mm_cvtpi8_ps(m1), 16.f, -16.f, 55.f, 127.f); // 4-way convert int8s from lowest end of m64 to float in a m128. /*M64*/aeq64(_mm_cvtps_pi16(c), 0x0002000200040004ULL); // 4-way convert floats to int16s in a m64. /*M64*/aeq64(_mm_cvtps_pi32(c), 0x0000000400000004ULL); // 2-way convert two lowest floats to int32s in a m64. /*M64*/aeq64(_mm_cvtps_pi8(c), 0x0000000002020404ULL); // 4-way convert floats to int8s in a m64, zero higher half of the returned m64. /*M64*/aeq(_mm_cvtpu16_ps(m1), 255.f , 32769.f, 4336.f, 14207.f); // 4-way convert uint16s to floats, return in a m128. /*M64*/aeq(_mm_cvtpu8_ps(m1), 16.f, 240.f, 55.f, 127.f); // 4-way convert uint8s from lowest end of m64 to float in a m128. #endif aeq(_mm_cvtsi64_ss(c, -9223372036854775808ULL), 1.5f, 2.5f, 3.5f, -9223372036854775808.f); // Convert single int64 to float, store in lowest channel of m128, and pass three higher channel unchanged. Assert(_mm_cvtss_f32(c) == 4.5f); // Extract lowest channel of m128 to a plain old float. Assert(_mm_cvtss_si64(f) == -9223372036854775808ULL); // Convert lowest channel of m128 from float to int64. #ifdef TEST_M64 /*M64*/aeq64(_mm_cvtt_ps2pi(e), 0x0000000200000003ULL); aeq64(_mm_cvtt_ps2pi(f), 0xfffffffe80000000ULL); // Truncating conversion from two lowest floats of m128 to int32s, return in a m64. #endif Assert(_mm_cvttss_si32(e) == 3); // Truncating conversion from the lowest float of a m128 to int32. Assert( _mm_cvtt_ss2si(e) == 3); // _mm_cvtt_ss2si is an alias to _mm_cvttss_si32. #ifdef TEST_M64 /*M64*/aeq64(_mm_cvttps_pi32(c), 0x0000000300000004ULL); // Truncating conversion from two lowest floats of m128 to m64. #endif Assert(_mm_cvttss_si64(f) == -9223372036854775808ULL); // Truncating conversion from lowest channel of m128 from float to int64. #ifndef __EMSCRIPTEN__ // TODO: Not implemented. // SSE1 General support: unsigned int mask = _MM_GET_EXCEPTION_MASK(); _MM_SET_EXCEPTION_MASK(mask); unsigned int flushZeroMode = _MM_GET_FLUSH_ZERO_MODE(); _MM_SET_FLUSH_ZERO_MODE(flushZeroMode); unsigned int roundingMode = _MM_GET_ROUNDING_MODE(); _MM_SET_ROUNDING_MODE(roundingMode); unsigned int csr = _mm_getcsr(); _mm_setcsr(csr); unsigned char dummyData[4096]; _mm_prefetch(dummyData, _MM_HINT_T0); _mm_prefetch(dummyData, _MM_HINT_T1); _mm_prefetch(dummyData, _MM_HINT_T2); _mm_prefetch(dummyData, _MM_HINT_NTA); _mm_sfence(); #endif // SSE1 Misc instructions: #ifdef TEST_M64 /*M64*/Assert(_mm_movemask_pi8(m1) == 100); // Return int with eight lowest bits set depending on the highest bits of the 8 uint8 input channels of the m64. /*M64*/Assert( _m_pmovmskb(m1) == 100); // _m_pmovmskb is an alias to _mm_movemask_pi8. #endif Assert(_mm_movemask_ps(_mm_set_ps(-1.f, 0.f, 1.f, NAN)) == 8); Assert(_mm_movemask_ps(_mm_set_ps(-INFINITY, -0.f, INFINITY, -INFINITY)) == 13); // Return int with four lowest bits set depending on the highest bits of the 4 m128 input channels. // SSE1 Probability/Statistics instructions: #ifdef TEST_M64 /*M64*/aeq64(_mm_avg_pu16(m1, m2), 0x7FEE9D4D43A234C8ULL); // 4-way average uint16s. /*M64*/aeq64( _m_pavgw(m1, m2), 0x7FEE9D4D43A234C8ULL); // _m_pavgw is an alias to _mm_avg_pu16. /*M64*/aeq64(_mm_avg_pu8(m1, m2), 0x7FEE9D4D43A23548ULL); // 8-way average uint8s. /*M64*/aeq64( _m_pavgb(m1, m2), 0x7FEE9D4D43A23548ULL); // _m_pavgb is an alias to _mm_avg_pu8. // SSE1 Special Math instructions: /*M64*/aeq64(_mm_max_pi16(m1, m2), 0xFFBA987654377FULL); // 4-way average uint16s. /*M64*/aeq64( _m_pmaxsw(m1, m2), 0xFFBA987654377FULL); // _m_pmaxsw is an alias to _mm_max_pi16. /*M64*/aeq64(_mm_max_pu8(m1, m2), 0xFEFFBA9876F0377FULL); // 4-way average uint16s. /*M64*/aeq64( _m_pmaxub(m1, m2), 0xFEFFBA9876F0377FULL); // _m_pmaxub is an alias to _mm_max_pu8. /*M64*/aeq64(_mm_min_pi16(m1, m2), 0xFEDC800110F03210ULL); // 4-way average uint16s. /*M64*/aeq64( _m_pminsw(m1, m2), 0xFEDC800110F03210ULL); // is an alias to _mm_min_pi16. /*M64*/aeq64(_mm_min_pu8(m1, m2), 0xDC800110543210ULL); // 4-way average uint16s. /*M64*/aeq64( _m_pminub(m1, m2), 0xDC800110543210ULL); // is an alias to _mm_min_pu8. #endif // a = [8, 6, 4, 2], b = [1, 2, 3, 4] aeq(_mm_max_ps(a, b), 8.f, 6.f, 4.f, 4.f); // 4-wide max. aeq(_mm_max_ss(a, _mm_set1_ps(100.f)), 8.f, 6.f, 4.f, 100.f); // Scalar max, pass three highest unchanged. aeq(_mm_min_ps(a, b), 1.f, 2.f, 3.f, 2.f); // 4-wide min. aeq(_mm_min_ss(a, _mm_set1_ps(-100.f)), 8.f, 6.f, 4.f, -100.f); // Scalar min, pass three highest unchanged. // SSE1 Swizzle instructions: #ifdef TEST_M64 /*M64*/Assert(_mm_extract_pi16(m1, 1) == 4336); // Extract the given int16 channel from a m64. /*M64*/Assert( _m_pextrw(m1, 1) == 4336); // _m_pextrw is an alias to _mm_extract_pi16. /*M64*/aeq64(_mm_insert_pi16(m1, 0xABCD, 1), 0xFF8001ABCD377FULL); // Insert a int16 to a specific channel of a m64. /*M64*/aeq64( _m_pinsrw(m1, 0xABCD, 1), 0xFF8001ABCD377FULL); // _m_pinsrw is an alias to _mm_insert_pi16. /*M64*/aeq64(_mm_shuffle_pi16(m1, _MM_SHUFFLE(1, 0, 3, 2)), 0x10F0377F00FF8001ULL); // Shuffle int16s around in the 4 channels of the m64. /*M64*/aeq64( _m_pshufw(m1, _MM_SHUFFLE(1, 0, 3, 2)), 0x10F0377F00FF8001ULL); // _m_pshufw is an alias to _mm_shuffle_pi16. #endif aeq(_mm_shuffle_ps(a, b, _MM_SHUFFLE(1, 0, 3, 2)), 3.f, 4.f, 8.f, 6.f); aeq(_mm_unpackhi_ps(a, b), 1.f , 8.f, 2.f, 6.f); aeq(_mm_unpacklo_ps(a, b), 3.f , 4.f, 4.f, 2.f); // Transposing a matrix via the xmmintrin.h-provided intrinsic. __m128 c0 = a; // [8, 6, 4, 2] __m128 c1 = b; // [1, 2, 3, 4] __m128 c2 = get_c(); // [1.5, 2.5, 3.5, 4.5] __m128 c3 = get_d(); // [8.5, 6.5, 4.5, 2.5] _MM_TRANSPOSE4_PS(c0, c1, c2, c3); aeq(c0, 2.5f, 4.5f, 4.f, 2.f); aeq(c1, 4.5f, 3.5f, 3.f, 4.f); aeq(c2, 6.5f, 2.5f, 2.f, 6.f); aeq(c3, 8.5f, 1.5f, 1.f, 8.f); // All done! if (numFailures == 0) printf("Success!\n"); else printf("%d tests failed!\n", numFailures); }
void process( struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid, const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out) { const int filters = dt_image_flipped_filter(&piece->pipe->image); dt_iop_highlights_data_t *data = (dt_iop_highlights_data_t *)piece->data; const float clip = data->clip * fminf(piece->pipe->processed_maximum[0], fminf(piece->pipe->processed_maximum[1], piece->pipe->processed_maximum[2])); // const int ch = piece->colors; if(dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) || !filters) { const __m128 clipm = _mm_set1_ps(clip); #ifdef _OPENMP #pragma omp parallel for schedule(dynamic) default(none) shared(ovoid, ivoid, roi_in, roi_out, data, piece) #endif for(int j=0; j<roi_out->height; j++) { float *out = (float *)ovoid + (size_t)4*roi_out->width*j; float *in = (float *)ivoid + (size_t)4*roi_in->width*j; for(int i=0; i<roi_out->width; i++) { _mm_stream_ps(out, _mm_min_ps(clipm, _mm_set_ps(in[3],in[2],in[1],in[0]))); in += 4; out += 4; } } _mm_sfence(); return; } switch(data->mode) { case DT_IOP_HIGHLIGHTS_INPAINT: // a1ex's (magiclantern) idea of color inpainting: { const float clips[4] = { 0.987*data->clip * piece->pipe->processed_maximum[0], 0.987*data->clip * piece->pipe->processed_maximum[1], 0.987*data->clip * piece->pipe->processed_maximum[2], clip}; #ifdef _OPENMP #pragma omp parallel for schedule(dynamic) default(none) shared(ovoid, ivoid, roi_in, roi_out, data, piece) #endif for(int j=0; j<roi_out->height; j++) { _interpolate_color(ivoid, ovoid, roi_out, 0, 1, j, clips, filters, 0); _interpolate_color(ivoid, ovoid, roi_out, 0, -1, j, clips, filters, 1); } // up/down directions #ifdef _OPENMP #pragma omp parallel for schedule(dynamic) default(none) shared(ovoid, ivoid, roi_in, roi_out, data, piece) #endif for(int i=0; i<roi_out->width; i++) { _interpolate_color(ivoid, ovoid, roi_out, 1, 1, i, clips, filters, 2); _interpolate_color(ivoid, ovoid, roi_out, 1, -1, i, clips, filters, 3); } break; } case DT_IOP_HIGHLIGHTS_LCH: #ifdef _OPENMP #pragma omp parallel for schedule(dynamic) default(none) shared(ovoid, ivoid, roi_in, roi_out, data, piece) #endif for(int j=0; j<roi_out->height; j++) { float *out = (float *)ovoid + (size_t)roi_out->width*j; float *in = (float *)ivoid + (size_t)roi_out->width*j; for(int i=0; i<roi_out->width; i++) { if(i==0 || i==roi_out->width-1 || j==0 || j==roi_out->height-1) { // fast path for border out[0] = in[0]; } else { // analyse one bayer block to get same number of rggb pixels each time const float near_clip = 0.96f*clip; const float post_clip = 1.10f*clip; float blend = 0.0f; float mean = 0.0f; for(int jj=0; jj<=1; jj++) { for(int ii=0; ii<=1; ii++) { const float val = in[(size_t)jj*roi_out->width + ii]; mean += val*0.25f; blend += (fminf(post_clip, val) - near_clip)/(post_clip-near_clip); } } blend = CLAMP(blend, 0.0f, 1.0f); if(blend > 0) { // recover: out[0] = blend*mean + (1.f-blend)*in[0]; } else out[0] = in[0]; } out ++; in ++; } } break; default: case DT_IOP_HIGHLIGHTS_CLIP: { const __m128 clipm = _mm_set1_ps(clip); const size_t n = (size_t)roi_out->height*roi_out->width; float *const out = (float *)ovoid; float *const in = (float *)ivoid; #ifdef _OPENMP #pragma omp parallel for schedule(static) default(none) #endif for(int j=0; j<n; j+=4) _mm_stream_ps(out+j, _mm_min_ps(clipm, _mm_load_ps(in+j))); _mm_sfence(); // lets see if there's a non-multiple of four rest to process: if(n & 3) for(size_t j=n&~3u; j<n; j++) out[j] = MIN(clip, in[j]); break; } } if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height); }
int sse3_ChirpData_ak8( sah_complex * cx_DataArray, sah_complex * cx_ChirpDataArray, int chirp_rate_ind, double chirp_rate, int ul_NumDataPoints, double sample_rate ) { #ifdef USE_MANUAL_CALLSTACK call_stack.enter("sse3_ChirpData_ak8()"); #endif int i; if (chirp_rate_ind == 0) { memcpy(cx_ChirpDataArray, cx_DataArray, (int)ul_NumDataPoints * sizeof(sah_complex) ); #ifdef USE_MANUAL_CALLSTACK call_stack.exit(); #endif return 0; } int vEnd; double srate = chirp_rate * 0.5 / (sample_rate * sample_rate); __m128d rate = _mm_set1_pd(chirp_rate * 0.5 / (sample_rate * sample_rate)); __m128d roundVal = _mm_set1_pd(srate >= 0.0 ? TWO_TO_52 : -TWO_TO_52); __m128d DFOUR = _mm_set_pd(4.0, 4.0); // main vectorised loop vEnd = ul_NumDataPoints - (ul_NumDataPoints & 3); __m128d di1 = _mm_set_pd(2.0, 0.0); // set time patterns for eventual moveldup/movehdup __m128d di2 = _mm_set_pd(3.0, 1.0); for (i = 0; i < vEnd; i += 4) { const float *d = (const float *) (cx_DataArray + i); float *cd = (float *) (cx_ChirpDataArray + i); __m128d a1, a2; __m128 d1, d2; __m128 cd1, cd2; __m128 td1, td2; __m128 x; __m128 y; __m128 z; __m128 s; __m128 c; __m128 m; // load the signal to be chirped d1 = _mm_load_ps(d); d2 = _mm_load_ps(d+4); // calculate the input angle a1 = _mm_mul_pd(_mm_mul_pd(di1, di1), rate); a2 = _mm_mul_pd(_mm_mul_pd(di2, di2), rate); // update times for next di1 = _mm_add_pd(di1, DFOUR); di2 = _mm_add_pd(di2, DFOUR); // reduce the angle to the range (-0.5, 0.5) a1 = _mm_sub_pd(a1, _mm_sub_pd(_mm_add_pd(a1, roundVal), roundVal)); a2 = _mm_sub_pd(a2, _mm_sub_pd(_mm_add_pd(a2, roundVal), roundVal)); // convert pair of packed double into packed single x = _mm_movelh_ps(_mm_cvtpd_ps(a1), _mm_cvtpd_ps(a2)); // 3 1 2 0 // square to the range [0, 0.25) y = _mm_mul_ps(x, x); // perform the initial polynomial approximations, Estrin's method z = _mm_mul_ps(y, y); s = _mm_mul_ps(_mm_add_ps(_mm_mul_ps(_mm_add_ps(_mm_mul_ps(y, SS4F), SS3F), z), _mm_add_ps(_mm_mul_ps(y, SS2F), SS1F)), x); c = _mm_add_ps(_mm_mul_ps(_mm_add_ps(_mm_mul_ps(y, CC3F), CC2F), z), _mm_add_ps(_mm_mul_ps(y, CC1F), ONE)); // perform first angle doubling x = _mm_sub_ps(_mm_mul_ps(c, c), _mm_mul_ps(s, s)); y = _mm_mul_ps(_mm_mul_ps(s, c), TWO); // calculate scaling factor to correct the magnitude m = _mm_sub_ps(_mm_sub_ps(TWO, _mm_mul_ps(x, x)), _mm_mul_ps(y, y)); // perform second angle doubling c = _mm_sub_ps(_mm_mul_ps(x, x), _mm_mul_ps(y, y)); s = _mm_mul_ps(_mm_mul_ps(y, x), TWO); // correct the magnitude (final sine / cosine approximations) c = _mm_mul_ps(c, m); // c3 c1 c2 c0 s = _mm_mul_ps(s, m); // chirp the data cd1 = _mm_moveldup_ps(c); // c1 c1 c0 c0 cd2 = _mm_movehdup_ps(c); // c3 c3 c2 c2 cd1 = _mm_mul_ps(cd1, d1); // c1.i1 c1.r1 c0.i0 c0.r0 cd2 = _mm_mul_ps(cd2, d2); // c3.i3 c3.r3 c2.i2 c2.r2 d1 = _mm_shuffle_ps(d1, d1, 0xb1); d2 = _mm_shuffle_ps(d2, d2, 0xb1); td1 = _mm_moveldup_ps(s); td2 = _mm_movehdup_ps(s); td1 = _mm_mul_ps(td1, d1); td2 = _mm_mul_ps(td2, d2); cd1 = _mm_addsub_ps(cd1, td1); cd2 = _mm_addsub_ps(cd2, td2); // store chirped values _mm_stream_ps(cd, cd1); _mm_stream_ps(cd+4, cd2); } // handle tail elements with scalar code for (; i < ul_NumDataPoints; ++i) { double angle = srate * i * i * 0.5; double s = sin(angle); double c = cos(angle); float re = cx_DataArray[i][0]; float im = cx_DataArray[i][1]; cx_ChirpDataArray[i][0] = re * c - im * s; cx_ChirpDataArray[i][1] = re * s + im * c; } analysis_state.FLOP_counter+=12.0*ul_NumDataPoints; #ifdef USE_MANUAL_CALLSTACK call_stack.exit(); #endif return 0; }
void process_sse2(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid, const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out) { const dt_iop_graduatednd_data_t *data = (dt_iop_graduatednd_data_t *)piece->data; const int ch = piece->colors; const int ix = (roi_in->x); const int iy = (roi_in->y); const float iw = piece->buf_in.width * roi_out->scale; const float ih = piece->buf_in.height * roi_out->scale; const float hw = iw / 2.0; const float hh = ih / 2.0; const float hw_inv = 1.0 / hw; const float hh_inv = 1.0 / hh; const float v = (-data->rotation / 180) * M_PI; const float sinv = sin(v); const float cosv = cos(v); const float filter_radie = sqrt((hh * hh) + (hw * hw)) / hh; const float offset = data->offset / 100.0 * 2; float color[3]; hsl2rgb(color, data->hue, data->saturation, 0.5); if(data->density < 0) for(int l = 0; l < 3; l++) color[l] = 1.0 - color[l]; #if 1 const float filter_compression = 1.0 / filter_radie / (1.0 - (0.5 + (data->compression / 100.0) * 0.9 / 2.0)) * 0.5; #else const float compression = data->compression / 100.0f; const float t = 1.0f - .8f / (.8f + compression); const float c = 1.0f + 1000.0f * powf(4.0, compression); #endif if(data->density > 0) { #ifdef _OPENMP #pragma omp parallel for default(none) shared(roi_out, color, data, ivoid, ovoid) schedule(static) #endif for(int y = 0; y < roi_out->height; y++) { size_t k = (size_t)roi_out->width * y * ch; const float *in = (float *)ivoid + k; float *out = (float *)ovoid + k; float length = (sinv * (-1.0 + ix * hw_inv) - cosv * (-1.0 + (iy + y) * hh_inv) - 1.0 + offset) * filter_compression; const float length_inc = sinv * hw_inv * filter_compression; __m128 c = _mm_set_ps(0, color[2], color[1], color[0]); __m128 c1 = _mm_sub_ps(_mm_set1_ps(1.0f), c); for(int x = 0; x < roi_out->width; x++, in += ch, out += ch) { #if 1 // !!! approximation is ok only when highest density is 8 // for input x = (data->density * CLIP( 0.5+length ), calculate 2^x as (e^(ln2*x/8))^8 // use exp2f approximation to calculate e^(ln2*x/8) // in worst case - density==8,CLIP(0.5-length) == 1.0 it gives 0.6% of error const float t = 0.693147181f /* ln2 */ * (data->density * CLIP(0.5f + length) / 8.0f); float d1 = t * t * 0.5f; float d2 = d1 * t * 0.333333333f; float d3 = d2 * t * 0.25f; float d = 1 + t + d1 + d2 + d3; /* taylor series for e^x till x^4 */ // printf("%d %d %f\n",y,x,d); __m128 density = _mm_set1_ps(d); density = _mm_mul_ps(density, density); density = _mm_mul_ps(density, density); density = _mm_mul_ps(density, density); #else // use fair exp2f __m128 density = _mm_set1_ps(exp2f(data->density * CLIP(0.5f + length))); #endif /* max(0,in / (c + (1-c)*density)) */ _mm_stream_ps(out, _mm_max_ps(_mm_set1_ps(0.0f), _mm_div_ps(_mm_load_ps(in), _mm_add_ps(c, _mm_mul_ps(c1, density))))); length += length_inc; } } } else { #ifdef _OPENMP #pragma omp parallel for default(none) shared(roi_out, color, data, ivoid, ovoid) schedule(static) #endif for(int y = 0; y < roi_out->height; y++) { size_t k = (size_t)roi_out->width * y * ch; const float *in = (float *)ivoid + k; float *out = (float *)ovoid + k; float length = (sinv * (-1.0f + ix * hw_inv) - cosv * (-1.0f + (iy + y) * hh_inv) - 1.0f + offset) * filter_compression; const float length_inc = sinv * hw_inv * filter_compression; __m128 c = _mm_set_ps(0, color[2], color[1], color[0]); __m128 c1 = _mm_sub_ps(_mm_set1_ps(1.0f), c); for(int x = 0; x < roi_out->width; x++, in += ch, out += ch) { #if 1 // !!! approximation is ok only when lowest density is -8 // for input x = (-data->density * CLIP( 0.5-length ), calculate 2^x as (e^(ln2*x/8))^8 // use exp2f approximation to calculate e^(ln2*x/8) // in worst case - density==-8,CLIP(0.5-length) == 1.0 it gives 0.6% of error const float t = 0.693147181f /* ln2 */ * (-data->density * CLIP(0.5f - length) / 8.0f); float d1 = t * t * 0.5f; float d2 = d1 * t * 0.333333333f; float d3 = d2 * t * 0.25f; float d = 1 + t + d1 + d2 + d3; /* taylor series for e^x till x^4 */ __m128 density = _mm_set1_ps(d); density = _mm_mul_ps(density, density); density = _mm_mul_ps(density, density); density = _mm_mul_ps(density, density); #else __m128 density = _mm_set1_ps(exp2f(-data->density * CLIP(0.5f - length))); #endif /* max(0,in * (c + (1-c)*density)) */ _mm_stream_ps(out, _mm_max_ps(_mm_set1_ps(0.0f), _mm_mul_ps(_mm_load_ps(in), _mm_add_ps(c, _mm_mul_ps(c1, density))))); length += length_inc; } } } _mm_sfence(); if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height); }
void process(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid, const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out) { const int filters = dt_image_filter(&piece->pipe->image); uint8_t (*const xtrans)[6] = self->dev->image_storage.xtrans; dt_iop_temperature_data_t *d = (dt_iop_temperature_data_t *)piece->data; if(!dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) && filters == 9u) { // xtrans float mosaiced #ifdef _OPENMP #pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid, d) schedule(static) #endif for(int j = 0; j < roi_out->height; j++) { const float *in = ((float *)ivoid) + (size_t)j * roi_out->width; float *out = ((float *)ovoid) + (size_t)j * roi_out->width; for(int i = 0; i < roi_out->width; i++, out++, in++) *out = *in * d->coeffs[FCxtrans(j, i, roi_out, xtrans)]; } } else if(!dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) && filters) { // bayer float mosaiced #ifdef _OPENMP #pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid, d) schedule(static) #endif for(int j = 0; j < roi_out->height; j++) { const float *in = ((float *)ivoid) + (size_t)j * roi_out->width; float *out = ((float *)ovoid) + (size_t)j * roi_out->width; int i = 0; int alignment = ((4 - (j * roi_out->width & (4 - 1))) & (4 - 1)); // process unaligned pixels for(; i < alignment; i++, out++, in++) *out = *in * d->coeffs[FC(j + roi_out->y, i + roi_out->x, filters)]; const __m128 coeffs = _mm_set_ps(d->coeffs[FC(j + roi_out->y, roi_out->x + i + 3, filters)], d->coeffs[FC(j + roi_out->y, roi_out->x + i + 2, filters)], d->coeffs[FC(j + roi_out->y, roi_out->x + i + 1, filters)], d->coeffs[FC(j + roi_out->y, roi_out->x + i, filters)]); // process aligned pixels with SSE for(; i < roi_out->width - (4 - 1); i += 4, in += 4, out += 4) { const __m128 input = _mm_load_ps(in); const __m128 multiplied = _mm_mul_ps(input, coeffs); _mm_stream_ps(out, multiplied); } // process the rest for(; i < roi_out->width; i++, out++, in++) *out = *in * d->coeffs[FC(j + roi_out->y, i + roi_out->x, filters)]; } _mm_sfence(); } else { // non-mosaiced const int ch = piece->colors; const __m128 coeffs = _mm_set_ps(1.0f, d->coeffs[2], d->coeffs[1], d->coeffs[0]); #ifdef _OPENMP #pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid, d) schedule(static) #endif for(int k = 0; k < roi_out->height; k++) { const float *in = ((float *)ivoid) + (size_t)ch * k * roi_out->width; float *out = ((float *)ovoid) + (size_t)ch * k * roi_out->width; for(int j = 0; j < roi_out->width; j++, in += ch, out += ch) { const __m128 input = _mm_load_ps(in); const __m128 multiplied = _mm_mul_ps(input, coeffs); _mm_stream_ps(out, multiplied); } } _mm_sfence(); if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height); } for(int k = 0; k < 3; k++) piece->pipe->processed_maximum[k] = d->coeffs[k] * piece->pipe->processed_maximum[k]; }
void process(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid, const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out) { float *in; float *out; dt_iop_zonesystem_gui_data_t *g = NULL; dt_iop_zonesystem_data_t *data = (dt_iop_zonesystem_data_t *)piece->data; const int width = roi_out->width; const int height = roi_out->height; if(self->dev->gui_attached && piece->pipe->type == DT_DEV_PIXELPIPE_PREVIEW) { g = (dt_iop_zonesystem_gui_data_t *)self->gui_data; dt_pthread_mutex_lock(&g->lock); if(g->in_preview_buffer == NULL || g->out_preview_buffer == NULL || g->preview_width != width || g->preview_height != height) { g_free(g->in_preview_buffer); g_free(g->out_preview_buffer); g->in_preview_buffer = g_malloc_n((size_t)width * height, sizeof(guchar)); g->out_preview_buffer = g_malloc_n((size_t)width * height, sizeof(guchar)); g->preview_width = width; g->preview_height = height; } dt_pthread_mutex_unlock(&g->lock); } /* calculate zonemap */ const int size = data->size; float zonemap[MAX_ZONE_SYSTEM_SIZE] = { -1 }; _iop_zonesystem_calculate_zonemap(data, zonemap); const int ch = piece->colors; /* process the image */ in = (float *)ivoid; out = (float *)ovoid; const float rzscale = (size - 1) / 100.0f; float zonemap_offset[MAX_ZONE_SYSTEM_SIZE] = { -1 }; float zonemap_scale[MAX_ZONE_SYSTEM_SIZE] = { -1 }; // precompute scale and offset for(int k = 0; k < size - 1; k++) zonemap_scale[k] = (zonemap[k + 1] - zonemap[k]) * (size - 1); for(int k = 0; k < size - 1; k++) zonemap_offset[k] = 100.0f * ((k + 1) * zonemap[k] - k * zonemap[k + 1]); #ifdef _OPENMP #pragma omp parallel for default(none) shared(in, out, zonemap_scale, zonemap_offset) schedule(static) #endif for(int j = 0; j < height; j++) for(int i = 0; i < width; i++) { /* remap lightness into zonemap and apply lightness */ const float *inp = in + ch * ((size_t)j * width + i); float *outp = out + ch * ((size_t)j * width + i); const int rz = CLAMPS(inp[0] * rzscale, 0, size - 2); // zone index const float zs = ((rz > 0) ? (zonemap_offset[rz] / inp[0]) : 0) + zonemap_scale[rz]; _mm_stream_ps(outp, _mm_mul_ps(_mm_load_ps(inp), _mm_set1_ps(zs))); } _mm_sfence(); if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, width, height); /* if gui and have buffer lets gaussblur and fill buffer with zone indexes */ if(self->dev->gui_attached && g && g->in_preview_buffer && g->out_preview_buffer) { float Lmax[] = { 100.0f }; float Lmin[] = { 0.0f }; /* setup gaussian kernel */ const int radius = 8; const float sigma = 2.5 * (radius * roi_in->scale / piece->iscale); dt_gaussian_t *gauss = dt_gaussian_init(width, height, 1, Lmax, Lmin, sigma, DT_IOP_GAUSSIAN_ZERO); float *tmp = g_malloc_n((size_t)width * height, sizeof(float)); if(gauss && tmp) { #ifdef _OPENMP #pragma omp parallel for default(none) shared(ivoid, tmp) schedule(static) #endif for(size_t k = 0; k < (size_t)width * height; k++) tmp[k] = ((float *)ivoid)[ch * k]; dt_gaussian_blur(gauss, tmp, tmp); /* create zonemap preview for input */ dt_pthread_mutex_lock(&g->lock); #ifdef _OPENMP #pragma omp parallel for default(none) shared(tmp, g) schedule(static) #endif for(size_t k = 0; k < (size_t)width * height; k++) { g->in_preview_buffer[k] = CLAMPS(tmp[k] * (size - 1) / 100.0f, 0, size - 2); } dt_pthread_mutex_unlock(&g->lock); #ifdef _OPENMP #pragma omp parallel for default(none) shared(ovoid, tmp) schedule(static) #endif for(size_t k = 0; k < (size_t)width * height; k++) tmp[k] = ((float *)ovoid)[ch * k]; dt_gaussian_blur(gauss, tmp, tmp); /* create zonemap preview for output */ dt_pthread_mutex_lock(&g->lock); #ifdef _OPENMP #pragma omp parallel for default(none) shared(tmp, g) schedule(static) #endif for(size_t k = 0; k < (size_t)width * height; k++) { g->out_preview_buffer[k] = CLAMPS(tmp[k] * (size - 1) / 100.0f, 0, size - 2); } dt_pthread_mutex_unlock(&g->lock); } g_free(tmp); if(gauss) dt_gaussian_free(gauss); } }
static void gui_update_from_coeffs(dt_iop_module_t *self) { dt_iop_invert_gui_data_t *g = (dt_iop_invert_gui_data_t *)self->gui_data; dt_iop_invert_params_t *p = (dt_iop_invert_params_t *)self->params; GdkRGBA color = (GdkRGBA){.red = p->color[0], .green = p->color[1], .blue = p->color[2], .alpha = 1.0 }; const dt_image_t *img = &self->dev->image_storage; if(img->flags & DT_IMAGE_4BAYER) { float rgb[4]; for(int k = 0; k < 4; k++) rgb[k] = p->color[k]; dt_colorspaces_cygm_to_rgb(rgb, 1, g->CAM_to_RGB); color.red = rgb[0]; color.green = rgb[1]; color.blue = rgb[2]; } gtk_color_chooser_set_rgba(GTK_COLOR_CHOOSER(g->colorpicker), &color); } static gboolean draw(GtkWidget *widget, cairo_t *cr, dt_iop_module_t *self) { if(darktable.gui->reset) return FALSE; if(self->picked_color_max[0] < 0.0f) return FALSE; if(self->request_color_pick == DT_REQUEST_COLORPICK_OFF) return FALSE; static float old[4] = { 0.0f, 0.0f, 0.0f, 0.0f }; const float *grayrgb = self->picked_color; if(grayrgb[0] == old[0] && grayrgb[1] == old[1] && grayrgb[2] == old[2] && grayrgb[3] == old[3]) return FALSE; for(int k = 0; k < 4; k++) old[k] = grayrgb[k]; dt_iop_invert_params_t *p = self->params; for(int k = 0; k < 4; k++) p->color[k] = grayrgb[k]; darktable.gui->reset = 1; gui_update_from_coeffs(self); darktable.gui->reset = 0; dt_dev_add_history_item(darktable.develop, self, TRUE); return FALSE; } static void colorpicker_callback(GtkColorButton *widget, dt_iop_module_t *self) { if(self->dt->gui->reset) return; dt_iop_invert_gui_data_t *g = (dt_iop_invert_gui_data_t *)self->gui_data; dt_iop_invert_params_t *p = (dt_iop_invert_params_t *)self->params; // turn off the other color picker so that this tool actually works ... gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(g->picker), FALSE); GdkRGBA c; gtk_color_chooser_get_rgba(GTK_COLOR_CHOOSER(widget), &c); p->color[0] = c.red; p->color[1] = c.green; p->color[2] = c.blue; const dt_image_t *img = &self->dev->image_storage; if(img->flags & DT_IMAGE_4BAYER) { dt_colorspaces_rgb_to_cygm(p->color, 1, g->RGB_to_CAM); } dt_dev_add_history_item(darktable.develop, self, TRUE); } void process(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, const void *const ivoid, void *const ovoid, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out) { const dt_iop_invert_data_t *const d = (dt_iop_invert_data_t *)piece->data; const float *const m = piece->pipe->dsc.processed_maximum; const float film_rgb_f[4] = { d->color[0] * m[0], d->color[1] * m[1], d->color[2] * m[2], d->color[3] * m[3] }; // FIXME: it could be wise to make this a NOP when picking colors. not sure about that though. // if(self->request_color_pick){ // do nothing // } const uint32_t filters = piece->pipe->dsc.filters; const uint8_t(*const xtrans)[6] = (const uint8_t(*const)[6])piece->pipe->dsc.xtrans; const float *const in = (const float *const)ivoid; float *const out = (float *const)ovoid; if(filters == 9u) { // xtrans float mosaiced #ifdef _OPENMP #pragma omp parallel for SIMD() default(none) schedule(static) collapse(2) #endif for(int j = 0; j < roi_out->height; j++) { for(int i = 0; i < roi_out->width; i++) { const size_t p = (size_t)j * roi_out->width + i; out[p] = CLAMP(film_rgb_f[FCxtrans(j, i, roi_out, xtrans)] - in[p], 0.0f, 1.0f); } } for(int k = 0; k < 4; k++) piece->pipe->dsc.processed_maximum[k] = 1.0f; } else if(filters) { // bayer float mosaiced #ifdef _OPENMP #pragma omp parallel for SIMD() default(none) schedule(static) collapse(2) #endif for(int j = 0; j < roi_out->height; j++) { for(int i = 0; i < roi_out->width; i++) { const size_t p = (size_t)j * roi_out->width + i; out[p] = CLAMP(film_rgb_f[FC(j + roi_out->y, i + roi_out->x, filters)] - in[p], 0.0f, 1.0f); } } for(int k = 0; k < 4; k++) piece->pipe->dsc.processed_maximum[k] = 1.0f; } else { // non-mosaiced const int ch = piece->colors; #ifdef _OPENMP #pragma omp parallel for SIMD() default(none) schedule(static) collapse(2) #endif for(size_t k = 0; k < (size_t)ch * roi_out->width * roi_out->height; k += ch) { for(int c = 0; c < 3; c++) { const size_t p = (size_t)k + c; out[p] = d->color[c] - in[p]; } } if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height); } } #if defined(__SSE__) void process_sse2(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, const void *const ivoid, void *const ovoid, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out) { dt_iop_invert_data_t *d = (dt_iop_invert_data_t *)piece->data; const float *const m = piece->pipe->dsc.processed_maximum; const float film_rgb_f[4] = { d->color[0] * m[0], d->color[1] * m[1], d->color[2] * m[2], d->color[3] * m[3] }; // FIXME: it could be wise to make this a NOP when picking colors. not sure about that though. // if(self->request_color_pick){ // do nothing // } const uint32_t filters = piece->pipe->dsc.filters; const uint8_t(*const xtrans)[6] = (const uint8_t(*const)[6])piece->pipe->dsc.xtrans; if(filters == 9u) { // xtrans float mosaiced const __m128 val_min = _mm_setzero_ps(); const __m128 val_max = _mm_set1_ps(1.0f); #ifdef _OPENMP #pragma omp parallel for default(none) schedule(static) #endif for(int j = 0; j < roi_out->height; j++) { const float *in = ((float *)ivoid) + (size_t)j * roi_out->width; float *out = ((float *)ovoid) + (size_t)j * roi_out->width; int i = 0; int alignment = ((4 - (j * roi_out->width & (4 - 1))) & (4 - 1)); // process unaligned pixels for(; i < alignment && i < roi_out->width; i++, out++, in++) *out = CLAMP(film_rgb_f[FCxtrans(j, i, roi_out, xtrans)] - *in, 0.0f, 1.0f); const __m128 film[3] = { _mm_set_ps(film_rgb_f[FCxtrans(j, i + 3, roi_out, xtrans)], film_rgb_f[FCxtrans(j, i + 2, roi_out, xtrans)], film_rgb_f[FCxtrans(j, i + 1, roi_out, xtrans)], film_rgb_f[FCxtrans(j, i + 0, roi_out, xtrans)]), _mm_set_ps(film_rgb_f[FCxtrans(j, i + 7, roi_out, xtrans)], film_rgb_f[FCxtrans(j, i + 6, roi_out, xtrans)], film_rgb_f[FCxtrans(j, i + 5, roi_out, xtrans)], film_rgb_f[FCxtrans(j, i + 4, roi_out, xtrans)]), _mm_set_ps(film_rgb_f[FCxtrans(j, i + 11, roi_out, xtrans)], film_rgb_f[FCxtrans(j, i + 10, roi_out, xtrans)], film_rgb_f[FCxtrans(j, i + 9, roi_out, xtrans)], film_rgb_f[FCxtrans(j, i + 8, roi_out, xtrans)]) }; // process aligned pixels with SSE for(int c = 0; c < 3 && i < roi_out->width - (4 - 1); c++, i += 4, in += 4, out += 4) { __m128 v; v = _mm_load_ps(in); v = _mm_sub_ps(film[c], v); v = _mm_min_ps(v, val_max); v = _mm_max_ps(v, val_min); _mm_stream_ps(out, v); } // process the rest for(; i < roi_out->width; i++, out++, in++) *out = CLAMP(film_rgb_f[FCxtrans(j, i, roi_out, xtrans)] - *in, 0.0f, 1.0f); } _mm_sfence(); for(int k = 0; k < 4; k++) piece->pipe->dsc.processed_maximum[k] = 1.0f; } else if(filters) { // bayer float mosaiced const __m128 val_min = _mm_setzero_ps(); const __m128 val_max = _mm_set1_ps(1.0f); #ifdef _OPENMP #pragma omp parallel for default(none) schedule(static) #endif for(int j = 0; j < roi_out->height; j++) { const float *in = ((float *)ivoid) + (size_t)j * roi_out->width; float *out = ((float *)ovoid) + (size_t)j * roi_out->width; int i = 0; int alignment = ((4 - (j * roi_out->width & (4 - 1))) & (4 - 1)); // process unaligned pixels for(; i < alignment && i < roi_out->width; i++, out++, in++) *out = CLAMP(film_rgb_f[FC(j + roi_out->y, i + roi_out->x, filters)] - *in, 0.0f, 1.0f); const __m128 film = _mm_set_ps(film_rgb_f[FC(j + roi_out->y, roi_out->x + i + 3, filters)], film_rgb_f[FC(j + roi_out->y, roi_out->x + i + 2, filters)], film_rgb_f[FC(j + roi_out->y, roi_out->x + i + 1, filters)], film_rgb_f[FC(j + roi_out->y, roi_out->x + i, filters)]); // process aligned pixels with SSE for(; i < roi_out->width - (4 - 1); i += 4, in += 4, out += 4) { const __m128 input = _mm_load_ps(in); const __m128 subtracted = _mm_sub_ps(film, input); _mm_stream_ps(out, _mm_max_ps(_mm_min_ps(subtracted, val_max), val_min)); } // process the rest for(; i < roi_out->width; i++, out++, in++) *out = CLAMP(film_rgb_f[FC(j + roi_out->y, i + roi_out->x, filters)] - *in, 0.0f, 1.0f); } _mm_sfence(); for(int k = 0; k < 4; k++) piece->pipe->dsc.processed_maximum[k] = 1.0f; } else { // non-mosaiced const int ch = piece->colors; const __m128 film = _mm_set_ps(1.0f, d->color[2], d->color[1], d->color[0]); #ifdef _OPENMP #pragma omp parallel for default(none) schedule(static) #endif for(int k = 0; k < roi_out->height; k++) { const float *in = ((float *)ivoid) + (size_t)ch * k * roi_out->width; float *out = ((float *)ovoid) + (size_t)ch * k * roi_out->width; for(int j = 0; j < roi_out->width; j++, in += ch, out += ch) { const __m128 input = _mm_load_ps(in); const __m128 subtracted = _mm_sub_ps(film, input); _mm_stream_ps(out, subtracted); } } _mm_sfence(); if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height); } }
/* Fast remote SCI copy for systems with write-combining enabled. This is the version using SSE instructions to copy 128 Byte blocks, and flushes after 32 Byte. */ void _mpid_smi_sse32_memcpy(void *dest, const void *src, size_t size) { char* a = (char*) src; char* b = (char*) dest; size_t j = 0; __m128 xmm[8]; /* Align the destination to a 64 Byte boundary */ for(; (j < size) && (((size_t) &b[j]) % 64 != 0); j++) ((char*) b)[j] = ((char*) a)[j]; // Loads two cache lines of data to a location closer to the processor. _mm_prefetch(a+j, _MM_HINT_NTA); _mm_prefetch(a+j+64, _MM_HINT_NTA); /* copy 128 byte per loop */ for (; (j+128) < size; j+=128) { // Loads two cache lines of data to a location closer to the processor. _mm_prefetch(a+j+128, _MM_HINT_NTA); _mm_prefetch(a+j+192, _MM_HINT_NTA); /* load 128 Byte into xmm register */ xmm[0] = _mm_load_ps((float*) &a[j]); xmm[1] = _mm_load_ps((float*) &a[j+16]); xmm[2] = _mm_load_ps((float*) &a[j+32]); xmm[3] = _mm_load_ps((float*) &a[j+48]); xmm[4] = _mm_load_ps((float*) &a[j+64]); xmm[5] = _mm_load_ps((float*) &a[j+80]); xmm[6] = _mm_load_ps((float*) &a[j+96]); xmm[7] = _mm_load_ps((float*) &a[j+112]); /* store 32 byte */ _mm_stream_ps((float*) &b[j], xmm[0]); _mm_stream_ps((float*) &b[j+16], xmm[1]); /* flush the write-combine buffer */ _mm_sfence(); /* store 32 byte */ _mm_stream_ps((float*) &b[j+32], xmm[2]); _mm_stream_ps((float*) &b[j+48], xmm[3]); /* flush the write-combine buffer */ _mm_sfence(); /* store 32 byte */ _mm_stream_ps((float*) &b[j+64], xmm[4]); _mm_stream_ps((float*) &b[j+80], xmm[5]); /* flush the write-combine buffer */ _mm_sfence(); /* store 32 byte */ _mm_stream_ps((float*) &b[j+96], xmm[6]); _mm_stream_ps((float*) &b[j+112], xmm[7]); /* flush the write-combine buffer */ _mm_sfence(); } /* copy tail */ for(; j<size; j++) ((char*) b)[j] = ((char*) a)[j]; }
void dt_interpolation_resample( const struct dt_interpolation* itor, float *out, const dt_iop_roi_t* const roi_out, const int32_t out_stride, const float* const in, const dt_iop_roi_t* const roi_in, const int32_t in_stride) { int* hindex = NULL; int* hlength = NULL; float* hkernel = NULL; int* vindex = NULL; int* vlength = NULL; float* vkernel = NULL; int* vmeta = NULL; int r; debug_info( "resampling %p (%dx%d@%dx%d scale %f) -> %p (%dx%d@%dx%d scale %f)\n", in, roi_in->width, roi_in->height, roi_in->x, roi_in->y, roi_in->scale, out, roi_out->width, roi_out->height, roi_out->x, roi_out->y, roi_out->scale); // Fast code path for 1:1 copy, only cropping area can change if (roi_out->scale == 1.f) { const int x0 = roi_out->x*4*sizeof(float); const int l = roi_out->width*4*sizeof(float); #if DEBUG_RESAMPLING_TIMING int64_t ts_resampling = getts(); #endif #ifdef _OPENMP #pragma omp parallel for default(none) shared(out) #endif for (int y=0; y<roi_out->height; y++) { float* i = (float*)((char*)in + in_stride*(y + roi_out->y) + x0); float* o = (float*)((char*)out + out_stride*y); memcpy(o, i, l); } #if DEBUG_RESAMPLING_TIMING ts_resampling = getts() - ts_resampling; fprintf(stderr, "resampling %p plan:0us resampling:%"PRId64"us\n", in, ts_resampling); #endif // All done, so easy case return; } // Generic non 1:1 case... much more complicated :D #if DEBUG_RESAMPLING_TIMING int64_t ts_plan = getts(); #endif // Prepare resampling plans once and for all r = prepare_resampling_plan(itor, roi_in->width, roi_in->x, roi_out->width, roi_out->x, roi_out->scale, &hlength, &hkernel, &hindex, NULL); if (r) { goto exit; } r = prepare_resampling_plan(itor, roi_in->height, roi_in->y, roi_out->height, roi_out->y, roi_out->scale, &vlength, &vkernel, &vindex, &vmeta); if (r) { goto exit; } #if DEBUG_RESAMPLING_TIMING ts_plan = getts() - ts_plan; #endif #if DEBUG_RESAMPLING_TIMING int64_t ts_resampling = getts(); #endif // Process each output line #ifdef _OPENMP #pragma omp parallel for default(none) shared(out, hindex, hlength, hkernel, vindex, vlength, vkernel, vmeta) #endif for (int oy=0; oy<roi_out->height; oy++) { // Initialize column resampling indexes int vlidx = vmeta[3*oy + 0]; // V(ertical) L(ength) I(n)d(e)x int vkidx = vmeta[3*oy + 1]; // V(ertical) K(ernel) I(n)d(e)x int viidx = vmeta[3*oy + 2]; // V(ertical) I(ndex) I(n)d(e)x // Initialize row resampling indexes int hlidx = 0; // H(orizontal) L(ength) I(n)d(e)x int hkidx = 0; // H(orizontal) K(ernel) I(n)d(e)x int hiidx = 0; // H(orizontal) I(ndex) I(n)d(e)x // Number of lines contributing to the output line int vl = vlength[vlidx++]; // V(ertical) L(ength) // Process each output column for (int ox=0; ox < roi_out->width; ox++) { debug_extra("output %p [% 4d % 4d]\n", out, ox, oy); // This will hold the resulting pixel __m128 vs = _mm_setzero_ps(); // Number of horizontal samples contributing to the output int hl = hlength[hlidx++]; // H(orizontal) L(ength) for (int iy=0; iy < vl; iy++) { // This is our input line const float* i = (float*)((char*)in + in_stride*vindex[viidx++]); __m128 vhs = _mm_setzero_ps(); for (int ix=0; ix< hl; ix++) { // Apply the precomputed filter kernel int baseidx = hindex[hiidx++]*4; float htap = hkernel[hkidx++]; __m128 vhtap = _mm_set_ps1(htap); vhs = _mm_add_ps(vhs, _mm_mul_ps(*(__m128*)&i[baseidx], vhtap)); } // Accumulate contribution from this line float vtap = vkernel[vkidx++]; __m128 vvtap = _mm_set_ps1(vtap); vs = _mm_add_ps(vs, _mm_mul_ps(vhs, vvtap)); // Reset horizontal resampling context hkidx -= hl; hiidx -= hl; } // Output pixel is ready float* o = (float*)((char*)out + oy*out_stride + ox*4*sizeof(float)); _mm_stream_ps(o, vs); // Reset vertical resampling context viidx -= vl; vkidx -= vl; // Progress in horizontal context hiidx += hl; hkidx += hl; } // Progress in vertical context viidx += vl; vkidx += vl; } _mm_sfence(); #if DEBUG_RESAMPLING_TIMING ts_resampling = getts() - ts_resampling; fprintf(stderr, "resampling %p plan:%"PRId64"us resampling:%"PRId64"us\n", in, ts_plan, ts_resampling); #endif exit: /* Free the resampling plans. It's nasty to optimize allocs like that, but * it simplifies the code :-D. The length array is in fact the only memory * allocated. */ free(hlength); free(vlength); }
void process (struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, const void * const ivoid, void *ovoid, const dt_iop_roi_t *roi_in, const dt_iop_roi_t * const roi_out) { dt_develop_t *dev = self->dev; const int ch = piece->colors; // FIXME: turn off the module instead? if(!dev->overexposed.enabled || !dev->gui_attached) { memcpy(ovoid, ivoid, (size_t)roi_out->width*roi_out->height*sizeof(float)*ch); return; } const __m128 upper = _mm_set_ps(FLT_MAX, dev->overexposed.upper / 100.0f, dev->overexposed.upper / 100.0f, dev->overexposed.upper / 100.0f); const __m128 lower = _mm_set_ps(FLT_MAX, dev->overexposed.lower / 100.0f, dev->overexposed.lower / 100.0f, dev->overexposed.lower / 100.0f); const int colorscheme = dev->overexposed.colorscheme; const __m128 upper_color = _mm_load_ps(dt_iop_overexposed_colors[colorscheme][0]); const __m128 lower_color = _mm_load_ps(dt_iop_overexposed_colors[colorscheme][1]); #ifdef _OPENMP #pragma omp parallel for default(none) shared(ovoid) schedule(static) #endif for(int k=0; k<roi_out->height; k++) { const float *in = ((float *)ivoid) + (size_t)ch*k*roi_out->width; float *out = ((float *)ovoid) + (size_t)ch*k*roi_out->width; for (int j=0; j<roi_out->width; j++,in+=4,out+=4) { const __m128 pixel = _mm_load_ps(in); __m128 isoe = _mm_cmpge_ps(pixel, upper); isoe = _mm_or_ps(_mm_unpacklo_ps(isoe, isoe), _mm_unpackhi_ps(isoe, isoe)); isoe = _mm_or_ps(_mm_unpacklo_ps(isoe, isoe), _mm_unpackhi_ps(isoe, isoe)); __m128 isue = _mm_cmple_ps(pixel, lower); isue = _mm_and_ps(_mm_unpacklo_ps(isue, isue), _mm_unpackhi_ps(isue, isue)); isue = _mm_and_ps(_mm_unpacklo_ps(isue, isue), _mm_unpackhi_ps(isue, isue)); __m128 result = _mm_or_ps(_mm_andnot_ps(isoe, pixel), _mm_and_ps(isoe, upper_color)); result = _mm_or_ps(_mm_andnot_ps(isue, result), _mm_and_ps(isue, lower_color)); _mm_stream_ps(out, result); } } _mm_sfence(); if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height); }
// ============================================================================= // // sse3_vChirpData // version by: Alex Kan // http://tbp.berkeley.edu/~alexkan/seti/ // int sse3_ChirpData_ak( sah_complex * cx_DataArray, sah_complex * cx_ChirpDataArray, int chirp_rate_ind, double chirp_rate, int ul_NumDataPoints, double sample_rate ) { int i; #ifdef USE_MANUAL_CALLSTACK call_stack.enter("sse3_ChirpData_ak()"); #endif if (chirp_rate_ind == 0) { memcpy(cx_ChirpDataArray, cx_DataArray, (int)ul_NumDataPoints * sizeof(sah_complex) ); #ifdef USE_MANUAL_CALLSTACK call_stack.exit(); #endif return 0; } int vEnd; double srate = chirp_rate * 0.5 / (sample_rate * sample_rate); __m128d rate = _mm_set1_pd(chirp_rate * 0.5 / (sample_rate * sample_rate)); __m128d roundVal = _mm_set1_pd(srate >= 0.0 ? TWO_TO_52 : -TWO_TO_52); // main vectorised loop vEnd = ul_NumDataPoints - (ul_NumDataPoints & 3); for (i = 0; i < vEnd; i += 4) { const float *data = (const float *) (cx_DataArray + i); float *chirped = (float *) (cx_ChirpDataArray + i); __m128d di = _mm_set1_pd(i); __m128d a1 = _mm_add_pd(_mm_set_pd(1.0, 0.0), di); __m128d a2 = _mm_add_pd(_mm_set_pd(3.0, 2.0), di); __m128 d1, d2; __m128 cd1, cd2; __m128 td1, td2; __m128 x; __m128 y; __m128 s; __m128 c; __m128 m; // load the signal to be chirped prefetchnta((const void *)( data+32 )); d1 = _mm_load_ps(data); d2 = _mm_load_ps(data+4); // calculate the input angle a1 = _mm_mul_pd(_mm_mul_pd(a1, a1), rate); a2 = _mm_mul_pd(_mm_mul_pd(a2, a2), rate); // reduce the angle to the range (-0.5, 0.5) a1 = _mm_sub_pd(a1, _mm_sub_pd(_mm_add_pd(a1, roundVal), roundVal)); a2 = _mm_sub_pd(a2, _mm_sub_pd(_mm_add_pd(a2, roundVal), roundVal)); // convert pair of packed double into packed single x = _mm_movelh_ps(_mm_cvtpd_ps(a1), _mm_cvtpd_ps(a2)); // square to the range [0, 0.25) y = _mm_mul_ps(x, x); // perform the initial polynomial approximations s = _mm_mul_ps(_mm_add_ps(_mm_mul_ps(_mm_add_ps(_mm_mul_ps(_mm_add_ps(_mm_mul_ps(y, SS4), SS3), y), SS2), y), SS1), x); c = _mm_add_ps(_mm_mul_ps(_mm_add_ps(_mm_mul_ps(_mm_add_ps(_mm_mul_ps(y, CC3), CC2), y), CC1), y), ONE); // perform first angle doubling x = _mm_sub_ps(_mm_mul_ps(c, c), _mm_mul_ps(s, s)); y = _mm_mul_ps(_mm_mul_ps(s, c), TWO); // calculate scaling factor to correct the magnitude // m1 = vec_nmsub(y1, y1, vec_nmsub(x1, x1, TWO)); // m2 = vec_nmsub(y2, y2, vec_nmsub(x2, x2, TWO)); m = vec_recip3(_mm_add_ps(_mm_mul_ps(x, x), _mm_mul_ps(y, y))); // perform second angle doubling c = _mm_sub_ps(_mm_mul_ps(x, x), _mm_mul_ps(y, y)); s = _mm_mul_ps(_mm_mul_ps(y, x), TWO); // correct the magnitude (final sine / cosine approximations) s = _mm_mul_ps(s, m); c = _mm_mul_ps(c, m); // chirp the data cd1 = _mm_shuffle_ps(c, c, 0x50); cd2 = _mm_shuffle_ps(c, c, 0xfa); cd1 = _mm_mul_ps(cd1, d1); cd2 = _mm_mul_ps(cd2, d2); d1 = _mm_shuffle_ps(d1, d1, 0xb1); d2 = _mm_shuffle_ps(d2, d2, 0xb1); td1 = _mm_shuffle_ps(s, s, 0x50); td2 = _mm_shuffle_ps(s, s, 0xfa); td1 = _mm_mul_ps(td1, d1); td2 = _mm_mul_ps(td2, d2); cd1 = _mm_addsub_ps(cd1, td1); cd2 = _mm_addsub_ps(cd2, td2); // store chirped values _mm_stream_ps(chirped, cd1); _mm_stream_ps(chirped+4, cd2); } _mm_sfence(); // handle tail elements with scalar code for ( ; i < ul_NumDataPoints; ++i) { double angle = srate * i * i * 0.5; double s = sin(angle); double c = cos(angle); float re = cx_DataArray[i][0]; float im = cx_DataArray[i][1]; cx_ChirpDataArray[i][0] = re * c - im * s; cx_ChirpDataArray[i][1] = re * s + im * c; } analysis_state.FLOP_counter+=12.0*ul_NumDataPoints; #ifdef USE_MANUAL_CALLSTACK call_stack.exit(); #endif return 0; }
void process_sse2(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, const void *const ivoid, void *const ovoid, const dt_iop_roi_t *const roi_in, const dt_iop_roi_t *const roi_out) { const dt_iop_colorout_data_t *const d = (dt_iop_colorout_data_t *)piece->data; const int ch = piece->colors; const int gamutcheck = (d->mode == DT_PROFILE_GAMUTCHECK); if(d->type == DT_COLORSPACE_LAB) { memcpy(ovoid, ivoid, sizeof(float)*4*roi_out->width*roi_out->height); } else if(!isnan(d->cmatrix[0])) { // fprintf(stderr,"Using cmatrix codepath\n"); // convert to rgb using matrix #ifdef _OPENMP #pragma omp parallel for schedule(static) default(none) #endif for(int j = 0; j < roi_out->height; j++) { float *in = (float *)ivoid + (size_t)ch * roi_in->width * j; float *out = (float *)ovoid + (size_t)ch * roi_out->width * j; const __m128 m0 = _mm_set_ps(0.0f, d->cmatrix[6], d->cmatrix[3], d->cmatrix[0]); const __m128 m1 = _mm_set_ps(0.0f, d->cmatrix[7], d->cmatrix[4], d->cmatrix[1]); const __m128 m2 = _mm_set_ps(0.0f, d->cmatrix[8], d->cmatrix[5], d->cmatrix[2]); for(int i = 0; i < roi_out->width; i++, in += ch, out += ch) { const __m128 xyz = dt_Lab_to_XYZ_SSE(_mm_load_ps(in)); const __m128 t = _mm_add_ps(_mm_mul_ps(m0, _mm_shuffle_ps(xyz, xyz, _MM_SHUFFLE(0, 0, 0, 0))), _mm_add_ps(_mm_mul_ps(m1, _mm_shuffle_ps(xyz, xyz, _MM_SHUFFLE(1, 1, 1, 1))), _mm_mul_ps(m2, _mm_shuffle_ps(xyz, xyz, _MM_SHUFFLE(2, 2, 2, 2))))); _mm_stream_ps(out, t); } } _mm_sfence(); process_fastpath_apply_tonecurves(self, piece, ivoid, ovoid, roi_in, roi_out); } else { // fprintf(stderr,"Using xform codepath\n"); const __m128 outofgamutpixel = _mm_set_ps(0.0f, 1.0f, 1.0f, 0.0f); #ifdef _OPENMP #pragma omp parallel for schedule(static) default(none) #endif for(int k = 0; k < roi_out->height; k++) { const float *in = ((float *)ivoid) + (size_t)ch * k * roi_out->width; float *out = ((float *)ovoid) + (size_t)ch * k * roi_out->width; cmsDoTransform(d->xform, in, out, roi_out->width); if(gamutcheck) { for(int j = 0; j < roi_out->width; j++, out += 4) { const __m128 pixel = _mm_load_ps(out); __m128 ingamut = _mm_cmplt_ps(pixel, _mm_set_ps(-FLT_MAX, 0.0f, 0.0f, 0.0f)); ingamut = _mm_or_ps(_mm_unpacklo_ps(ingamut, ingamut), _mm_unpackhi_ps(ingamut, ingamut)); ingamut = _mm_or_ps(_mm_unpacklo_ps(ingamut, ingamut), _mm_unpackhi_ps(ingamut, ingamut)); const __m128 result = _mm_or_ps(_mm_and_ps(ingamut, outofgamutpixel), _mm_andnot_ps(ingamut, pixel)); _mm_stream_ps(out, result); } } } _mm_sfence(); } if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height); }
void process (struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid, const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out) { float *in; float *out; dt_iop_zonesystem_gui_data_t *g = NULL; dt_iop_zonesystem_data_t *data = (dt_iop_zonesystem_data_t*)piece->data; guchar *buffer = NULL; if( self->dev->gui_attached && piece->pipe->type == DT_DEV_PIXELPIPE_PREVIEW ) { g = (dt_iop_zonesystem_gui_data_t *)self->gui_data; dt_pthread_mutex_lock(&g->lock); if(g->preview_buffer) g_free (g->preview_buffer); buffer = g->preview_buffer = g_malloc (roi_in->width*roi_in->height); g->preview_width=roi_out->width; g->preview_height=roi_out->height; } /* calculate zonemap */ const int size = data->size; float zonemap[MAX_ZONE_SYSTEM_SIZE]= {-1}; _iop_zonesystem_calculate_zonemap (data, zonemap); const int ch = piece->colors; /* if gui and have buffer lets gaussblur and fill buffer with zone indexes */ if( self->dev->gui_attached && g && buffer) { /* setup gaussian kernel */ const int radius = 8; const float _r = ceilf(radius * roi_in->scale / piece->iscale); const int rad = MIN(radius, _r); const int wd = 2*rad+1; float mat[wd*wd]; float *m; const float sigma2 = (2.5*2.5)*(radius*roi_in->scale/piece->iscale)*(radius*roi_in->scale/piece->iscale); float weight = 0.0f; memset(mat, 0, wd*wd*sizeof(float)); m = mat; for(int l=-rad; l<=rad; l++) for(int k=-rad; k<=rad; k++,m++) weight += *m = expf(- (l*l + k*k)/(2.f*sigma2)); m = mat; for(int l=-rad; l<=rad; l++) for(int k=-rad; k<=rad; k++,m++) *m /= weight; /* gauss blur the L channel */ #ifdef _OPENMP #pragma omp parallel for default(none) private(in, out, m) shared(mat, ivoid, ovoid, roi_out, roi_in) schedule(static) #endif for(int j=rad; j<roi_out->height-rad; j++) { in = ((float *)ivoid) + ch*(j*roi_in->width + rad); out = ((float *)ovoid) + ch*(j*roi_out->width + rad); for(int i=rad; i<roi_out->width-rad; i++) { for(int c=0; c<3; c++) out[c] = 0.0f; float sum = 0.0; m = mat; for(int l=-rad; l<=rad; l++) { float *inrow = in + ch*(l*roi_in->width-rad); for(int k=-rad; k<=rad; k++,inrow+=ch,m++) sum += *m * inrow[0]; } out[0] = sum; out += ch; in += ch; } } /* create zonemap preview */ // in = (float *)ivoid; out = (float *)ovoid; #ifdef _OPENMP #pragma omp parallel for default(none) shared(roi_out,out,buffer,g,zonemap) schedule(static) #endif for (int k=0; k<roi_out->width*roi_out->height; k++) { buffer[k] = _iop_zonesystem_zone_index_from_lightness (out[ch*k]/100.0f, zonemap, size); } dt_pthread_mutex_unlock(&g->lock); } /* process the image */ in = (float *)ivoid; out = (float *)ovoid; const float rzscale = (size-1)/100.0f; float zonemap_offset[MAX_ZONE_SYSTEM_SIZE]= {-1}; float zonemap_scale[MAX_ZONE_SYSTEM_SIZE]= {-1}; // precompute scale and offset for (int k=0; k < size-1; k++) zonemap_scale[k] = (zonemap[k+1]-zonemap[k])*(size-1); for (int k=0; k < size-1; k++) zonemap_offset[k] = 100.0f * ((k+1)*zonemap[k] - k*zonemap[k+1]) ; #ifdef _OPENMP #pragma omp parallel for default(none) shared(roi_out, in, out, zonemap_scale,zonemap_offset) schedule(static) #endif for (int j=0; j<roi_out->height; j++) for (int i=0; i<roi_out->width; i++) { /* remap lightness into zonemap and apply lightness */ const float *inp = in + ch*(j*roi_out->width+i); float *outp = out + ch*(j*roi_out->width+i); const int rz = CLAMPS(inp[0]*rzscale, 0, size-2); // zone index const float zs = ((rz > 0) ? (zonemap_offset[rz]/inp[0]) : 0) + zonemap_scale[rz]; _mm_stream_ps(outp,_mm_mul_ps(_mm_load_ps(inp),_mm_set1_ps(zs))); } _mm_sfence(); if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height); }
/* A vectorized version of the Voigt function using X86 SSE instructions */ void my_voigt(const float *damping, const float *frequency_offset, float *voigt_value, int N) { // coefficients of the rational approximation formula // to the complementary error function const __m128 A0 = _mm_set1_ps(122.607931777104326f); const __m128 A1 = _mm_set1_ps(214.382388694706425f); const __m128 A2 = _mm_set1_ps(181.928533092181549f); const __m128 A3 = _mm_set1_ps(93.155580458138441f); const __m128 A4 = _mm_set1_ps(30.180142196210589f); const __m128 A5 = _mm_set1_ps(5.912626209773153f); const __m128 A6 = _mm_set1_ps(0.564189583562615f); const __m128 B0 = _mm_set1_ps(122.60793177387535f); const __m128 B1 = _mm_set1_ps(352.730625110963558f); const __m128 B2 = _mm_set1_ps(457.334478783897737f); const __m128 B3 = _mm_set1_ps(348.703917719495792f); const __m128 B4 = _mm_set1_ps(170.354001821091472f); const __m128 B5 = _mm_set1_ps(53.992906912940207f); const __m128 B6 = _mm_set1_ps(10.479857114260399f); __m128 ivsigno; __m128 V; __m128 Z1_real; __m128 Z1_imag; __m128 Z2_real; __m128 Z2_imag; __m128 Z3_real; __m128 Z3_imag; __m128 Z4_real; __m128 Z4_imag; __m128 Z5_real; __m128 Z5_imag; __m128 Z6_real; __m128 Z6_imag; __m128 ZZ1_real; __m128 ZZ1_imag; __m128 ZZ2_real; __m128 ZZ2_imag; __m128 ZZ3_real; __m128 ZZ3_imag; __m128 ZZ4_real; __m128 ZZ4_imag; __m128 ZZ5_real; __m128 ZZ5_imag; __m128 ZZ6_real; __m128 ZZ6_imag; __m128 ZZ7_real; __m128 ZZ7_imag; __m128 division_factor; __m128 ZZZ_real; __m128 damp; __m128 offs; __m128 vval; __m128 one = _mm_set1_ps(1.0f); __m128 zero = _mm_set1_ps(0.0f); __m128 mone = _mm_set1_ps(-1.0f); __m128 half = _mm_set1_ps(-0.5f); __m128 mask; float *stmp = (float *) _mm_malloc(4*sizeof(float), 16); int i; for(i=0; i<N; i+=VECLEN){ _mm_prefetch((const char *)&damping[i+64], _MM_HINT_T0); _mm_prefetch((const char *)&frequency_offset[i+64], _MM_HINT_T0); damp = _mm_load_ps(&damping[i]); offs = _mm_load_ps(&frequency_offset[i]); mask = _mm_cmplt_ps(offs, zero); ivsigno = _mm_add_ps(_mm_and_ps(mask,mone),_mm_andnot_ps(mask,one)); V = _mm_mul_ps(ivsigno, offs); Z1_real = _mm_add_ps(_mm_mul_ps(A6, damp), A5); Z1_imag = _mm_mul_ps(A6, V); Z2_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(Z1_real, damp), _mm_mul_ps(Z1_imag, V)), A4); Z2_imag = _mm_add_ps(_mm_mul_ps(Z1_real, V), _mm_mul_ps(Z1_imag, damp)); Z3_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(Z2_real, damp), _mm_mul_ps(Z2_imag, V)), A3); Z3_imag = _mm_add_ps(_mm_mul_ps(Z2_real, V), _mm_mul_ps(Z2_imag, damp)); Z4_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(Z3_real, damp), _mm_mul_ps(Z3_imag, V)), A2); Z4_imag = _mm_add_ps(_mm_mul_ps(Z3_real, V), _mm_mul_ps(Z3_imag, damp)); Z5_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(Z4_real, damp), _mm_mul_ps(Z4_imag, V)), A1); Z5_imag = _mm_add_ps(_mm_mul_ps(Z4_real, V), _mm_mul_ps(Z4_imag, damp)); Z6_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(Z5_real, damp), _mm_mul_ps(Z5_imag, V)), A0); Z6_imag = _mm_add_ps(_mm_mul_ps(Z5_real, V), _mm_mul_ps(Z5_imag, damp)); ZZ1_real = _mm_add_ps(damp, B6); ZZ1_imag = V; ZZ2_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(ZZ1_real, damp), _mm_mul_ps(ZZ1_imag, V)), B5); ZZ2_imag = _mm_add_ps(_mm_mul_ps(ZZ1_real, V), _mm_mul_ps(ZZ1_imag, damp)); ZZ3_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(ZZ2_real, damp), _mm_mul_ps(ZZ2_imag, V)), B4); ZZ3_imag = _mm_add_ps(_mm_mul_ps(ZZ2_real, V), _mm_mul_ps(ZZ2_imag, damp)); ZZ4_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(ZZ3_real, damp), _mm_mul_ps(ZZ3_imag, V)), B3); ZZ4_imag = _mm_add_ps(_mm_mul_ps(ZZ3_real, V), _mm_mul_ps(ZZ3_imag, damp)); ZZ5_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(ZZ4_real, damp), _mm_mul_ps(ZZ4_imag, V)), B2); ZZ5_imag = _mm_add_ps(_mm_mul_ps(ZZ4_real, V), _mm_mul_ps(ZZ4_imag, damp)); ZZ6_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(ZZ5_real, damp), _mm_mul_ps(ZZ5_imag, V)), B1); ZZ6_imag = _mm_add_ps(_mm_mul_ps(ZZ5_real, V), _mm_mul_ps(ZZ5_imag, damp)); ZZ7_real = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(ZZ6_real, damp), _mm_mul_ps(ZZ6_imag, V)), B0); ZZ7_imag = _mm_add_ps(_mm_mul_ps(ZZ6_real, V), _mm_mul_ps(ZZ6_imag, damp)); division_factor = _mm_div_ps(one, _mm_add_ps(_mm_mul_ps(ZZ7_real, ZZ7_real), _mm_mul_ps(ZZ7_imag, ZZ7_imag))); ZZZ_real = _mm_mul_ps((_mm_add_ps(_mm_mul_ps(Z6_real, ZZ7_real), _mm_mul_ps(Z6_imag, ZZ7_imag))), division_factor); _mm_stream_ps(&voigt_value[i], ZZZ_real); } _mm_free(stmp); }
void process (struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid, const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out) { const dt_iop_colorout_data_t *const d = (dt_iop_colorout_data_t *)piece->data; const int ch = piece->colors; const int gamutcheck = (d->softproof_enabled == DT_SOFTPROOF_GAMUTCHECK); if(!isnan(d->cmatrix[0])) { //fprintf(stderr,"Using cmatrix codepath\n"); // convert to rgb using matrix #ifdef _OPENMP #pragma omp parallel for schedule(static) default(none) shared(roi_in,roi_out, ivoid, ovoid) #endif for(int j=0; j<roi_out->height; j++) { float *in = (float*)ivoid + ch*roi_in->width *j; float *out = (float*)ovoid + ch*roi_out->width*j; const __m128 m0 = _mm_set_ps(0.0f,d->cmatrix[6],d->cmatrix[3],d->cmatrix[0]); const __m128 m1 = _mm_set_ps(0.0f,d->cmatrix[7],d->cmatrix[4],d->cmatrix[1]); const __m128 m2 = _mm_set_ps(0.0f,d->cmatrix[8],d->cmatrix[5],d->cmatrix[2]); for(int i=0; i<roi_out->width; i++, in+=ch, out+=ch ) { const __m128 xyz = dt_Lab_to_XYZ_SSE(_mm_load_ps(in)); const __m128 t = _mm_add_ps(_mm_mul_ps(m0,_mm_shuffle_ps(xyz,xyz,_MM_SHUFFLE(0,0,0,0))),_mm_add_ps(_mm_mul_ps(m1,_mm_shuffle_ps(xyz,xyz,_MM_SHUFFLE(1,1,1,1))),_mm_mul_ps(m2,_mm_shuffle_ps(xyz,xyz,_MM_SHUFFLE(2,2,2,2))))); _mm_stream_ps(out,t); } } _mm_sfence(); // apply profile #ifdef _OPENMP #pragma omp parallel for schedule(static) default(none) shared(roi_in,roi_out, ivoid, ovoid) #endif for(int j=0; j<roi_out->height; j++) { float *in = (float*)ivoid + ch*roi_in->width *j; float *out = (float*)ovoid + ch*roi_out->width*j; for(int i=0; i<roi_out->width; i++, in+=ch, out+=ch ) { for(int i=0; i<3; i++) if (d->lut[i][0] >= 0.0f) { out[i] = (out[i] < 1.0f) ? lerp_lut(d->lut[i], out[i]) : dt_iop_eval_exp(d->unbounded_coeffs[i], out[i]); } } } } else { float *in = (float*)ivoid; float *out = (float*)ovoid; const int rowsize=roi_out->width * 3; //fprintf(stderr,"Using xform codepath\n"); #ifdef _OPENMP #pragma omp parallel for schedule(static) default(none) shared(out, roi_out, in) #endif for (int k=0; k<roi_out->height; k++) { float Lab[rowsize]; float rgb[rowsize]; const int m=(k*(roi_out->width*ch)); for (int l=0; l<roi_out->width; l++) { int li=3*l,ii=ch*l; Lab[li+0] = in[m+ii+0]; Lab[li+1] = in[m+ii+1]; Lab[li+2] = in[m+ii+2]; } cmsDoTransform (d->xform, Lab, rgb, roi_out->width); for (int l=0; l<roi_out->width; l++) { int oi=ch*l, ri=3*l; if(gamutcheck && (rgb[ri+0] < 0.0f || rgb[ri+1] < 0.0f || rgb[ri+2] < 0.0f)) { out[m+oi+0] = 0.0f; out[m+oi+1] = 1.0f; out[m+oi+2] = 1.0f; } else { out[m+oi+0] = rgb[ri+0]; out[m+oi+1] = rgb[ri+1]; out[m+oi+2] = rgb[ri+2]; } } } } if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height); }
void process (struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid, const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out) { const int filters = dt_image_flipped_filter(&piece->pipe->image); dt_iop_temperature_data_t *d = (dt_iop_temperature_data_t *)piece->data; if(!dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) && filters && piece->pipe->image.bpp != 4) { const float coeffsi[3] = {d->coeffs[0]/65535.0f, d->coeffs[1]/65535.0f, d->coeffs[2]/65535.0f}; #ifdef _OPENMP #pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid, d) schedule(static) #endif for(int j=0; j<roi_out->height; j++) { int i=0; const uint16_t *in = ((uint16_t *)ivoid) + j*roi_out->width; float *out = ((float*)ovoid) + j*roi_out->width; // process unaligned pixels for ( ; i < ((4-(j*roi_out->width & 3)) & 3) ; i++,out++,in++) *out = *in * coeffsi[FC(j+roi_out->y, i+roi_out->x, filters)]; const __m128 coeffs = _mm_set_ps(coeffsi[FC(j+roi_out->y, roi_out->x+i+3, filters)], coeffsi[FC(j+roi_out->y, roi_out->x+i+2, filters)], coeffsi[FC(j+roi_out->y, roi_out->x+i+1, filters)], coeffsi[FC(j+roi_out->y, roi_out->x+i , filters)]); // process aligned pixels with SSE for( ; i < roi_out->width - 3 ; i+=4,out+=4,in+=4) { _mm_stream_ps(out,_mm_mul_ps(coeffs,_mm_set_ps(in[3],in[2],in[1],in[0]))); } // process the rest for( ; i<roi_out->width; i++,out++,in++) *out = *in * coeffsi[FC(j+roi_out->y, i+roi_out->x, filters)]; } _mm_sfence(); } else if(!dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) && filters && piece->pipe->image.bpp == 4) { #ifdef _OPENMP #pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid, d) schedule(static) #endif for(int j=0; j<roi_out->height; j++) { const float *in = ((float *)ivoid) + j*roi_out->width; float *out = ((float*)ovoid) + j*roi_out->width; for(int i=0; i<roi_out->width; i++,out++,in++) *out = *in * d->coeffs[FC(j+roi_out->x, i+roi_out->y, filters)]; } } else { const int ch = piece->colors; #ifdef _OPENMP #pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid, d) schedule(static) #endif for(int k=0; k<roi_out->height; k++) { const float *in = ((float*)ivoid) + ch*k*roi_out->width; float *out = ((float*)ovoid) + ch*k*roi_out->width; for (int j=0; j<roi_out->width; j++,in+=ch,out+=ch) for(int c=0; c<3; c++) out[c] = in[c]*d->coeffs[c]; } } for(int k=0; k<3; k++) piece->pipe->processed_maximum[k] = d->coeffs[k] * piece->pipe->processed_maximum[k]; }
static gboolean draw(GtkWidget *widget, cairo_t *cr, dt_iop_module_t *self) { if(darktable.gui->reset) return FALSE; if(self->picked_color_max[0] < 0.0f) return FALSE; if(self->request_color_pick == DT_REQUEST_COLORPICK_OFF) return FALSE; dt_iop_invert_gui_data_t *g = (dt_iop_invert_gui_data_t *)self->gui_data; dt_iop_invert_params_t *p = (dt_iop_invert_params_t *)self->params; if(fabsf(p->color[0] - self->picked_color[0]) < 0.0001f && fabsf(p->color[1] - self->picked_color[1]) < 0.0001f && fabsf(p->color[2] - self->picked_color[2]) < 0.0001f) { // interrupt infinite loops return FALSE; } p->color[0] = self->picked_color[0]; p->color[1] = self->picked_color[1]; p->color[2] = self->picked_color[2]; GdkRGBA color = (GdkRGBA){.red = p->color[0], .green = p->color[1], .blue = p->color[2], .alpha = 1.0 }; gtk_color_chooser_set_rgba(GTK_COLOR_CHOOSER(g->colorpicker), &color); dt_dev_add_history_item(darktable.develop, self, TRUE); return FALSE; } static void colorpicker_callback(GtkColorButton *widget, dt_iop_module_t *self) { if(self->dt->gui->reset) return; dt_iop_invert_gui_data_t *g = (dt_iop_invert_gui_data_t *)self->gui_data; dt_iop_invert_params_t *p = (dt_iop_invert_params_t *)self->params; // turn off the other color picker so that this tool actually works ... gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(g->picker), FALSE); GdkRGBA c; gtk_color_chooser_get_rgba(GTK_COLOR_CHOOSER(widget), &c); p->color[0] = c.red; p->color[1] = c.green; p->color[2] = c.blue; dt_dev_add_history_item(darktable.develop, self, TRUE); } static int FC(const int row, const int col, const unsigned int filters) { return filters >> (((row << 1 & 14) + (col & 1)) << 1) & 3; } static uint8_t FCxtrans(const int row, const int col, const dt_iop_roi_t *const roi, uint8_t (*const xtrans)[6]) { return xtrans[(row + roi->y) % 6][(col + roi->x) % 6]; } void process(struct dt_iop_module_t *self, dt_dev_pixelpipe_iop_t *piece, void *ivoid, void *ovoid, const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out) { dt_iop_invert_data_t *d = (dt_iop_invert_data_t *)piece->data; const float *const m = piece->pipe->processed_maximum; const float film_rgb[3] = { d->color[0], d->color[1], d->color[2] }; const float film_rgb_f[3] = { d->color[0] * m[0], d->color[1] * m[1], d->color[2] * m[2] }; // FIXME: it could be wise to make this a NOP when picking colors. not sure about that though. // if(self->request_color_pick){ // do nothing // } const int filters = dt_image_filter(&piece->pipe->image); uint8_t (*const xtrans)[6] = self->dev->image_storage.xtrans; if(!dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) && (filters == 9u)) { // xtrans float mosaiced #ifdef _OPENMP #pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid) schedule(static) #endif for(int j = 0; j < roi_out->height; j++) { const float *in = ((float *)ivoid) + (size_t)j * roi_out->width; float *out = ((float *)ovoid) + (size_t)j * roi_out->width; for(int i = 0; i < roi_out->width; i++, out++, in++) *out = CLAMP(film_rgb_f[FCxtrans(j, i, roi_out, xtrans)] - *in, 0.0f, 1.0f); } for(int k = 0; k < 3; k++) piece->pipe->processed_maximum[k] = 1.0f; } else if(!dt_dev_pixelpipe_uses_downsampled_input(piece->pipe) && filters) { // bayer float mosaiced const __m128 val_min = _mm_setzero_ps(); const __m128 val_max = _mm_set1_ps(1.0f); #ifdef _OPENMP #pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid) schedule(static) #endif for(int j = 0; j < roi_out->height; j++) { const float *in = ((float *)ivoid) + (size_t)j * roi_out->width; float *out = ((float *)ovoid) + (size_t)j * roi_out->width; int i = 0; int alignment = ((4 - (j * roi_out->width & (4 - 1))) & (4 - 1)); // process unaligned pixels for(; i < alignment; i++, out++, in++) *out = CLAMP(film_rgb_f[FC(j + roi_out->y, i + roi_out->x, filters)] - *in, 0.0f, 1.0f); const __m128 film = _mm_set_ps(film_rgb_f[FC(j + roi_out->y, roi_out->x + i + 3, filters)], film_rgb_f[FC(j + roi_out->y, roi_out->x + i + 2, filters)], film_rgb_f[FC(j + roi_out->y, roi_out->x + i + 1, filters)], film_rgb_f[FC(j + roi_out->y, roi_out->x + i, filters)]); // process aligned pixels with SSE for(; i < roi_out->width - (4 - 1); i += 4, in += 4, out += 4) { const __m128 input = _mm_load_ps(in); const __m128 subtracted = _mm_sub_ps(film, input); _mm_stream_ps(out, _mm_max_ps(_mm_min_ps(subtracted, val_max), val_min)); } // process the rest for(; i < roi_out->width; i++, out++, in++) *out = CLAMP(film_rgb_f[FC(j + roi_out->y, i + roi_out->x, filters)] - *in, 0.0f, 1.0f); } _mm_sfence(); for(int k = 0; k < 3; k++) piece->pipe->processed_maximum[k] = 1.0f; } else { // non-mosaiced const int ch = piece->colors; const __m128 film = _mm_set_ps(1.0f, film_rgb[2], film_rgb[1], film_rgb[0]); #ifdef _OPENMP #pragma omp parallel for default(none) shared(roi_out, ivoid, ovoid) schedule(static) #endif for(int k = 0; k < roi_out->height; k++) { const float *in = ((float *)ivoid) + (size_t)ch * k * roi_out->width; float *out = ((float *)ovoid) + (size_t)ch * k * roi_out->width; for(int j = 0; j < roi_out->width; j++, in += ch, out += ch) { const __m128 input = _mm_load_ps(in); const __m128 subtracted = _mm_sub_ps(film, input); _mm_stream_ps(out, subtracted); } } _mm_sfence(); if(piece->pipe->mask_display) dt_iop_alpha_copy(ivoid, ovoid, roi_out->width, roi_out->height); } }