static force_inline uint32_t
bits_image_fetch_pixel_bilinear (bits_image_t *image,
pixman_fixed_t x,
pixman_fixed_t y,
get_pixel_t get_pixel)
{
pixman_repeat_t repeat_mode = image->common.repeat;
int width = image->width;
int height = image->height;
int x1, y1, x2, y2;
uint32_t tl, tr, bl, br;
int32_t distx, disty;
x1 = x - pixman_fixed_1 / 2;
y1 = y - pixman_fixed_1 / 2;
distx = pixman_fixed_to_bilinear_weight (x1);
disty = pixman_fixed_to_bilinear_weight (y1);
x1 = pixman_fixed_to_int (x1);
y1 = pixman_fixed_to_int (y1);
x2 = x1 + 1;
y2 = y1 + 1;
if (repeat_mode != PIXMAN_REPEAT_NONE)
{
repeat (repeat_mode, &x1, width);
repeat (repeat_mode, &y1, height);
repeat (repeat_mode, &x2, width);
repeat (repeat_mode, &y2, height);
tl = get_pixel (image, x1, y1, FALSE);
bl = get_pixel (image, x1, y2, FALSE);
tr = get_pixel (image, x2, y1, FALSE);
br = get_pixel (image, x2, y2, FALSE);
}
else
{
tl = get_pixel (image, x1, y1, TRUE);
tr = get_pixel (image, x2, y1, TRUE);
bl = get_pixel (image, x1, y2, TRUE);
br = get_pixel (image, x2, y2, TRUE);
}
return bilinear_interpolation (tl, tr, bl, br, distx, disty);
}
static uint32_t *
bits_image_fetch_bilinear_no_repeat_8888 (pixman_iter_t *iter,
const uint32_t *mask)
{
pixman_image_t * ima = iter->image;
int offset = iter->x;
int line = iter->y++;
int width = iter->width;
uint32_t * buffer = iter->buffer;
bits_image_t *bits = &ima->bits;
pixman_fixed_t x_top, x_bottom, x;
pixman_fixed_t ux_top, ux_bottom, ux;
pixman_vector_t v;
uint32_t top_mask, bottom_mask;
uint32_t *top_row;
uint32_t *bottom_row;
uint32_t *end;
uint32_t zero[2] = { 0, 0 };
uint32_t one = 1;
int y, y1, y2;
int disty;
int mask_inc;
int w;
/* reference point is the center of the pixel */
v.vector[0] = pixman_int_to_fixed (offset) + pixman_fixed_1 / 2;
v.vector[1] = pixman_int_to_fixed (line) + pixman_fixed_1 / 2;
v.vector[2] = pixman_fixed_1;
if (!pixman_transform_point_3d (bits->common.transform, &v))
return iter->buffer;
ux = ux_top = ux_bottom = bits->common.transform->matrix[0][0];
x = x_top = x_bottom = v.vector[0] - pixman_fixed_1/2;
y = v.vector[1] - pixman_fixed_1/2;
disty = pixman_fixed_to_bilinear_weight (y);
/* Load the pointers to the first and second lines from the source
* image that bilinear code must read.
*
* The main trick in this code is about the check if any line are
* outside of the image;
*
* When I realize that a line (any one) is outside, I change
* the pointer to a dummy area with zeros. Once I change this, I
* must be sure the pointer will not change, so I set the
* variables to each pointer increments inside the loop.
*/
y1 = pixman_fixed_to_int (y);
y2 = y1 + 1;
if (y1 < 0 || y1 >= bits->height)
{
top_row = zero;
x_top = 0;
ux_top = 0;
}
else
{
top_row = bits->bits + y1 * bits->rowstride;
x_top = x;
ux_top = ux;
}
if (y2 < 0 || y2 >= bits->height)
{
bottom_row = zero;
x_bottom = 0;
ux_bottom = 0;
}
else
{
bottom_row = bits->bits + y2 * bits->rowstride;
x_bottom = x;
ux_bottom = ux;
}
/* Instead of checking whether the operation uses the mast in
* each loop iteration, verify this only once and prepare the
* variables to make the code smaller inside the loop.
*/
if (!mask)
{
mask_inc = 0;
mask = &one;
}
else
{
/* If have a mask, prepare the variables to check it */
mask_inc = 1;
}
/* If both are zero, then the whole thing is zero */
if (top_row == zero && bottom_row == zero)
{
memset (buffer, 0, width * sizeof (uint32_t));
return iter->buffer;
}
else if (bits->format == PIXMAN_x8r8g8b8)
{
if (top_row == zero)
{
top_mask = 0;
bottom_mask = 0xff000000;
}
else if (bottom_row == zero)
{
top_mask = 0xff000000;
bottom_mask = 0;
}
else
{
top_mask = 0xff000000;
bottom_mask = 0xff000000;
}
}
else
{
top_mask = 0;
bottom_mask = 0;
}
end = buffer + width;
/* Zero fill to the left of the image */
while (buffer < end && x < pixman_fixed_minus_1)
{
*buffer++ = 0;
x += ux;
x_top += ux_top;
x_bottom += ux_bottom;
mask += mask_inc;
}
/* Left edge
*/
while (buffer < end && x < 0)
{
uint32_t tr, br;
int32_t distx;
tr = top_row[pixman_fixed_to_int (x_top) + 1] | top_mask;
br = bottom_row[pixman_fixed_to_int (x_bottom) + 1] | bottom_mask;
distx = pixman_fixed_to_bilinear_weight (x);
*buffer++ = bilinear_interpolation (0, tr, 0, br, distx, disty);
x += ux;
x_top += ux_top;
x_bottom += ux_bottom;
mask += mask_inc;
}
/* Main part */
w = pixman_int_to_fixed (bits->width - 1);
while (buffer < end && x < w)
{
if (*mask)
{
uint32_t tl, tr, bl, br;
int32_t distx;
tl = top_row [pixman_fixed_to_int (x_top)] | top_mask;
tr = top_row [pixman_fixed_to_int (x_top) + 1] | top_mask;
bl = bottom_row [pixman_fixed_to_int (x_bottom)] | bottom_mask;
br = bottom_row [pixman_fixed_to_int (x_bottom) + 1] | bottom_mask;
distx = pixman_fixed_to_bilinear_weight (x);
*buffer = bilinear_interpolation (tl, tr, bl, br, distx, disty);
}
buffer++;
x += ux;
x_top += ux_top;
x_bottom += ux_bottom;
mask += mask_inc;
}
/* Right Edge */
w = pixman_int_to_fixed (bits->width);
while (buffer < end && x < w)
{
if (*mask)
{
uint32_t tl, bl;
int32_t distx;
tl = top_row [pixman_fixed_to_int (x_top)] | top_mask;
bl = bottom_row [pixman_fixed_to_int (x_bottom)] | bottom_mask;
distx = pixman_fixed_to_bilinear_weight (x);
*buffer = bilinear_interpolation (tl, 0, bl, 0, distx, disty);
}
buffer++;
x += ux;
x_top += ux_top;
x_bottom += ux_bottom;
mask += mask_inc;
}
/* Zero fill to the left of the image */
while (buffer < end)
*buffer++ = 0;
return iter->buffer;
}
static force_inline void
bits_image_fetch_bilinear_affine (pixman_image_t * image,
int offset,
int line,
int width,
uint32_t * buffer,
const uint32_t * mask,
convert_pixel_t convert_pixel,
pixman_format_code_t format,
pixman_repeat_t repeat_mode)
{
pixman_fixed_t x, y;
pixman_fixed_t ux, uy;
pixman_vector_t v;
bits_image_t *bits = &image->bits;
int i;
/* reference point is the center of the pixel */
v.vector[0] = pixman_int_to_fixed (offset) + pixman_fixed_1 / 2;
v.vector[1] = pixman_int_to_fixed (line) + pixman_fixed_1 / 2;
v.vector[2] = pixman_fixed_1;
if (!pixman_transform_point_3d (image->common.transform, &v))
return;
ux = image->common.transform->matrix[0][0];
uy = image->common.transform->matrix[1][0];
x = v.vector[0];
y = v.vector[1];
for (i = 0; i < width; ++i)
{
int x1, y1, x2, y2;
uint32_t tl, tr, bl, br;
int32_t distx, disty;
int width = image->bits.width;
int height = image->bits.height;
const uint8_t *row1;
const uint8_t *row2;
if (mask && !mask[i])
goto next;
x1 = x - pixman_fixed_1 / 2;
y1 = y - pixman_fixed_1 / 2;
distx = pixman_fixed_to_bilinear_weight (x1);
disty = pixman_fixed_to_bilinear_weight (y1);
y1 = pixman_fixed_to_int (y1);
y2 = y1 + 1;
x1 = pixman_fixed_to_int (x1);
x2 = x1 + 1;
if (repeat_mode != PIXMAN_REPEAT_NONE)
{
uint32_t mask;
mask = PIXMAN_FORMAT_A (format)? 0 : 0xff000000;
repeat (repeat_mode, &x1, width);
repeat (repeat_mode, &y1, height);
repeat (repeat_mode, &x2, width);
repeat (repeat_mode, &y2, height);
row1 = (uint8_t *)bits->bits + bits->rowstride * 4 * y1;
row2 = (uint8_t *)bits->bits + bits->rowstride * 4 * y2;
tl = convert_pixel (row1, x1) | mask;
tr = convert_pixel (row1, x2) | mask;
bl = convert_pixel (row2, x1) | mask;
br = convert_pixel (row2, x2) | mask;
}
else
{
uint32_t mask1, mask2;
int bpp;
/* Note: PIXMAN_FORMAT_BPP() returns an unsigned value,
* which means if you use it in expressions, those
* expressions become unsigned themselves. Since
* the variables below can be negative in some cases,
* that will lead to crashes on 64 bit architectures.
*
* So this line makes sure bpp is signed
*/
bpp = PIXMAN_FORMAT_BPP (format);
if (x1 >= width || x2 < 0 || y1 >= height || y2 < 0)
{
buffer[i] = 0;
goto next;
}
if (y2 == 0)
{
row1 = zero;
mask1 = 0;
}
else
{
row1 = (uint8_t *)bits->bits + bits->rowstride * 4 * y1;
row1 += bpp / 8 * x1;
mask1 = PIXMAN_FORMAT_A (format)? 0 : 0xff000000;
}
if (y1 == height - 1)
{
row2 = zero;
mask2 = 0;
}
else
{
row2 = (uint8_t *)bits->bits + bits->rowstride * 4 * y2;
row2 += bpp / 8 * x1;
mask2 = PIXMAN_FORMAT_A (format)? 0 : 0xff000000;
}
if (x2 == 0)
{
tl = 0;
bl = 0;
}
else
{
tl = convert_pixel (row1, 0) | mask1;
bl = convert_pixel (row2, 0) | mask2;
}
if (x1 == width - 1)
{
tr = 0;
br = 0;
}
else
{
tr = convert_pixel (row1, 1) | mask1;
br = convert_pixel (row2, 1) | mask2;
}
}
buffer[i] = bilinear_interpolation (
tl, tr, bl, br, distx, disty);
next:
x += ux;
y += uy;
}
}
static uint32_t *
ssse3_fetch_bilinear_cover (pixman_iter_t *iter, const uint32_t *mask)
{
pixman_fixed_t fx, ux;
bilinear_info_t *info = iter->data;
line_t *line0, *line1;
int y0, y1;
int32_t dist_y;
__m128i vw;
int i;
fx = info->x;
ux = iter->image->common.transform->matrix[0][0];
y0 = pixman_fixed_to_int (info->y);
y1 = y0 + 1;
line0 = &info->lines[y0 & 0x01];
line1 = &info->lines[y1 & 0x01];
if (line0->y != y0)
{
ssse3_fetch_horizontal (
&iter->image->bits, line0, y0, fx, ux, iter->width);
}
if (line1->y != y1)
{
ssse3_fetch_horizontal (
&iter->image->bits, line1, y1, fx, ux, iter->width);
}
dist_y = pixman_fixed_to_bilinear_weight (info->y);
dist_y <<= (16 - BILINEAR_INTERPOLATION_BITS);
vw = _mm_set_epi16 (
dist_y, dist_y, dist_y, dist_y, dist_y, dist_y, dist_y, dist_y);
for (i = 0; i + 3 < iter->width; i += 4)
{
__m128i top0 = _mm_load_si128 ((__m128i *)(line0->buffer + i));
__m128i bot0 = _mm_load_si128 ((__m128i *)(line1->buffer + i));
__m128i top1 = _mm_load_si128 ((__m128i *)(line0->buffer + i + 2));
__m128i bot1 = _mm_load_si128 ((__m128i *)(line1->buffer + i + 2));
__m128i r0, r1, tmp, p;
r0 = _mm_mulhi_epu16 (
_mm_sub_epi16 (bot0, top0), vw);
tmp = _mm_cmplt_epi16 (bot0, top0);
tmp = _mm_and_si128 (tmp, vw);
r0 = _mm_sub_epi16 (r0, tmp);
r0 = _mm_add_epi16 (r0, top0);
r0 = _mm_srli_epi16 (r0, BILINEAR_INTERPOLATION_BITS);
/* r0: A0 R0 A1 R1 G0 B0 G1 B1 */
r0 = _mm_shuffle_epi32 (r0, _MM_SHUFFLE (2, 0, 3, 1));
/* r0: A1 R1 G1 B1 A0 R0 G0 B0 */
r1 = _mm_mulhi_epu16 (
_mm_sub_epi16 (bot1, top1), vw);
tmp = _mm_cmplt_epi16 (bot1, top1);
tmp = _mm_and_si128 (tmp, vw);
r1 = _mm_sub_epi16 (r1, tmp);
r1 = _mm_add_epi16 (r1, top1);
r1 = _mm_srli_epi16 (r1, BILINEAR_INTERPOLATION_BITS);
r1 = _mm_shuffle_epi32 (r1, _MM_SHUFFLE (2, 0, 3, 1));
/* r1: A3 R3 G3 B3 A2 R2 G2 B2 */
p = _mm_packus_epi16 (r0, r1);
_mm_storeu_si128 ((__m128i *)(iter->buffer + i), p);
}
while (i < iter->width)
{
__m128i top0 = _mm_load_si128 ((__m128i *)(line0->buffer + i));
__m128i bot0 = _mm_load_si128 ((__m128i *)(line1->buffer + i));
__m128i r0, tmp, p;
r0 = _mm_mulhi_epu16 (
_mm_sub_epi16 (bot0, top0), vw);
tmp = _mm_cmplt_epi16 (bot0, top0);
tmp = _mm_and_si128 (tmp, vw);
r0 = _mm_sub_epi16 (r0, tmp);
r0 = _mm_add_epi16 (r0, top0);
r0 = _mm_srli_epi16 (r0, BILINEAR_INTERPOLATION_BITS);
/* r0: A0 R0 A1 R1 G0 B0 G1 B1 */
r0 = _mm_shuffle_epi32 (r0, _MM_SHUFFLE (2, 0, 3, 1));
/* r0: A1 R1 G1 B1 A0 R0 G0 B0 */
p = _mm_packus_epi16 (r0, r0);
if (iter->width - i == 1)
{
*(uint32_t *)(iter->buffer + i) = _mm_cvtsi128_si32 (p);
i++;
}
else
{
_mm_storel_epi64 ((__m128i *)(iter->buffer + i), p);
i += 2;
}
}
info->y += iter->image->common.transform->matrix[1][1];
return iter->buffer;
}
