mp_limb_t
mpn_mul_1 (mp_ptr rp, mp_srcptr up, mp_size_t n, mp_limb_t vl)
{
mp_limb_t cy[n];
mp_limb_t a, b, r, s0, s1, c0, c1;
mp_size_t i;
int more_carries;
if (up == rp)
{
/* The algorithm used below cannot handle overlap. Handle it here by
making a temporary copy of the source vector, then call ourselves. */
mp_limb_t xp[n];
MPN_COPY (xp, up, n);
return mpn_mul_1 (rp, xp, n, vl);
}
a = up[0] * vl;
rp[0] = a;
cy[0] = 0;
/* Main multiply loop. Generate a raw accumulated output product in rp[]
and a carry vector in cy[]. */
#pragma _CRI ivdep
for (i = 1; i < n; i++)
{
a = up[i] * vl;
b = _int_mult_upper (up[i - 1], vl);
s0 = a + b;
c0 = ((a & b) | ((a | b) & ~s0)) >> 63;
rp[i] = s0;
cy[i] = c0;
}
/* Carry add loop. Add the carry vector cy[] to the raw sum rp[] and
store the new sum back to rp[0]. */
more_carries = 0;
#pragma _CRI ivdep
for (i = 2; i < n; i++)
{
r = rp[i];
c0 = cy[i - 1];
s0 = r + c0;
rp[i] = s0;
c0 = (r & ~s0) >> 63;
more_carries += c0;
}
/* If that second loop generated carry, handle that in scalar loop. */
if (more_carries)
{
mp_limb_t cyrec = 0;
/* Look for places where rp[k] is zero and cy[k-1] is non-zero.
These are where we got a recurrency carry. */
for (i = 2; i < n; i++)
{
r = rp[i];
c0 = (r == 0 && cy[i - 1] != 0);
s0 = r + cyrec;
rp[i] = s0;
c1 = (r & ~s0) >> 63;
cyrec = c0 | c1;
}
return _int_mult_upper (up[n - 1], vl) + cyrec + cy[n - 1];
}
return _int_mult_upper (up[n - 1], vl) + cy[n - 1];
}
void
mpn_sqr_basecase (mp_ptr rp,
mp_srcptr up, mp_size_t un)
{
mp_limb_t cy[un + un];
mp_limb_t ul;
mp_limb_t a, b, r, s0, s1, c0, c1;
mp_size_t i, j;
int more_carries;
for (i = 0; i < un + un; i++)
{
rp[i] = 0;
cy[i] = 0;
}
#pragma _CRI novector
for (j = 0; j < un; j++)
{
ul = up[j];
a = up[0] * ul;
r = rp[j];
s0 = a + r;
rp[j] = s0;
c0 = ((a & r) | ((a | r) & ~s0)) >> 63;
cy[j] += c0;
#pragma _CRI ivdep
for (i = 1; i < un; i++)
{
a = up[i] * ul;
b = _int_mult_upper (up[i - 1], ul);
s0 = a + b;
c0 = ((a & b) | ((a | b) & ~s0)) >> 63;
r = rp[j + i];
s1 = s0 + r;
rp[j + i] = s1;
c1 = ((s0 & r) | ((s0 | r) & ~s1)) >> 63;
cy[j + i] += c0 + c1;
}
rp[j + un] = _int_mult_upper (up[un - 1], ul);
}
more_carries = 0;
#pragma _CRI ivdep
for (i = 1; i < un + un; i++)
{
r = rp[i];
c0 = cy[i - 1];
s0 = r + c0;
rp[i] = s0;
c0 = (r & ~s0) >> 63;
more_carries += c0;
}
/* If that second loop generated carry, handle that in scalar loop. */
if (more_carries)
{
mp_limb_t cyrec = 0;
for (i = 1; i < un + un; i++)
{
r = rp[i];
c0 = (r < cy[i - 1]);
s0 = r + cyrec;
rp[i] = s0;
c1 = (r & ~s0) >> 63;
cyrec = c0 | c1;
}
}
}
