/* Computes {rp,MIN(rn,an+bn)} <- {ap,an}*{bp,bn} Mod(B^rn-1)
*
* The result is expected to be ZERO if and only if one of the operand
* already is. Otherwise the class [0] Mod(B^rn-1) is represented by
* B^rn-1. This should not be a problem if mulmod_bnm1 is used to
* combine results and obtain a natural number when one knows in
* advance that the final value is less than (B^rn-1).
* Moreover it should not be a problem if mulmod_bnm1 is used to
* compute the full product with an+bn <= rn, because this condition
* implies (B^an-1)(B^bn-1) < (B^rn-1) .
*
* Requires 0 < bn <= an <= rn and an + bn > rn/2
* Scratch need: rn + (need for recursive call OR rn + 4). This gives
*
* S(n) <= rn + MAX (rn + 4, S(n/2)) <= 2rn + 4
*/
void
mpn_mulmod_bnm1 (mp_ptr rp, mp_size_t rn, mp_srcptr ap, mp_size_t an, mp_srcptr bp, mp_size_t bn, mp_ptr tp)
{
ASSERT (0 < bn);
ASSERT (bn <= an);
ASSERT (an <= rn);
if ((rn & 1) != 0 || BELOW_THRESHOLD (rn, MULMOD_BNM1_THRESHOLD))
{
if (UNLIKELY (bn < rn))
{
if (UNLIKELY (an + bn <= rn))
{
mpn_mul (rp, ap, an, bp, bn);
}
else
{
mp_limb_t cy;
mpn_mul (tp, ap, an, bp, bn);
cy = mpn_add (rp, tp, rn, tp + rn, an + bn - rn);
MPN_INCR_U (rp, rn, cy);
}
}
else
mpn_bc_mulmod_bnm1 (rp, ap, bp, rn, tp);
}
else
{
mp_size_t n;
mp_limb_t cy;
mp_limb_t hi;
n = rn >> 1;
/* We need at least an + bn >= n, to be able to fit one of the
recursive products at rp. Requiring strict inequality makes
the coded slightly simpler. If desired, we could avoid this
restriction by initially halving rn as long as rn is even and
an + bn <= rn/2. */
ASSERT (an + bn > n);
/* Compute xm = a*b mod (B^n - 1), xp = a*b mod (B^n + 1)
and crt together as
x = -xp * B^n + (B^n + 1) * [ (xp + xm)/2 mod (B^n-1)]
*/
#define a0 ap
#define a1 (ap + n)
#define b0 bp
#define b1 (bp + n)
#define xp tp /* 2n + 2 */
/* am1 maybe in {xp, n} */
/* bm1 maybe in {xp + n, n} */
#define sp1 (tp + 2*n + 2)
/* ap1 maybe in {sp1, n + 1} */
/* bp1 maybe in {sp1 + n + 1, n + 1} */
{
mp_srcptr am1, bm1;
mp_size_t anm, bnm;
mp_ptr so;
bm1 = b0;
bnm = bn;
if (LIKELY (an > n))
{
am1 = xp;
cy = mpn_add (xp, a0, n, a1, an - n);
MPN_INCR_U (xp, n, cy);
anm = n;
so = xp + n;
if (LIKELY (bn > n))
{
bm1 = so;
cy = mpn_add (so, b0, n, b1, bn - n);
MPN_INCR_U (so, n, cy);
bnm = n;
so += n;
}
}
else
{
so = xp;
am1 = a0;
anm = an;
}
mpn_mulmod_bnm1 (rp, n, am1, anm, bm1, bnm, so);
}
{
int k;
mp_srcptr ap1, bp1;
mp_size_t anp, bnp;
bp1 = b0;
bnp = bn;
if (LIKELY (an > n)) {
ap1 = sp1;
cy = mpn_sub (sp1, a0, n, a1, an - n);
sp1[n] = 0;
MPN_INCR_U (sp1, n + 1, cy);
anp = n + ap1[n];
if (LIKELY (bn > n)) {
bp1 = sp1 + n + 1;
cy = mpn_sub (sp1 + n + 1, b0, n, b1, bn - n);
sp1[2*n+1] = 0;
MPN_INCR_U (sp1 + n + 1, n + 1, cy);
bnp = n + bp1[n];
}
} else {
ap1 = a0;
anp = an;
}
if (BELOW_THRESHOLD (n, MUL_FFT_MODF_THRESHOLD))
k=0;
else
{
int mask;
k = mpn_fft_best_k (n, 0);
mask = (1<<k) - 1;
while (n & mask) {k--; mask >>=1;};
}
if (k >= FFT_FIRST_K)
xp[n] = mpn_mul_fft (xp, n, ap1, anp, bp1, bnp, k);
else if (UNLIKELY (bp1 == b0))
{
ASSERT (anp + bnp <= 2*n+1);
ASSERT (anp + bnp > n);
ASSERT (anp >= bnp);
mpn_mul (xp, ap1, anp, bp1, bnp);
anp = anp + bnp - n;
ASSERT (anp <= n || xp[2*n]==0);
anp-= anp > n;
cy = mpn_sub (xp, xp, n, xp + n, anp);
xp[n] = 0;
MPN_INCR_U (xp, n+1, cy);
}
else
mpn_bc_mulmod_bnp1 (xp, ap1, bp1, n, xp);
}
/* Here the CRT recomposition begins.
xm <- (xp + xm)/2 = (xp + xm)B^n/2 mod (B^n-1)
Division by 2 is a bitwise rotation.
Assumes xp normalised mod (B^n+1).
The residue class [0] is represented by [B^n-1]; except when
both input are ZERO.
*/
#if HAVE_NATIVE_mpn_rsh1add_n || HAVE_NATIVE_mpn_rsh1add_nc
#if HAVE_NATIVE_mpn_rsh1add_nc
cy = mpn_rsh1add_nc(rp, rp, xp, n, xp[n]); /* B^n = 1 */
hi = cy << (GMP_NUMB_BITS - 1);
cy = 0;
/* next update of rp[n-1] will set cy = 1 only if rp[n-1]+=hi
overflows, i.e. a further increment will not overflow again. */
#else /* ! _nc */
cy = xp[n] + mpn_rsh1add_n(rp, rp, xp, n); /* B^n = 1 */
hi = (cy<<(GMP_NUMB_BITS-1))&GMP_NUMB_MASK; /* (cy&1) << ... */
cy >>= 1;
/* cy = 1 only if xp[n] = 1 i.e. {xp,n} = ZERO, this implies that
the rsh1add was a simple rshift: the top bit is 0. cy=1 => hi=0. */
#endif
#if GMP_NAIL_BITS == 0
add_ssaaaa(cy, rp[n-1], cy, rp[n-1], 0, hi);
#else
cy += (hi & rp[n-1]) >> (GMP_NUMB_BITS-1);
rp[n-1] ^= hi;
#endif
#else /* ! HAVE_NATIVE_mpn_rsh1add_n */
#if HAVE_NATIVE_mpn_add_nc
cy = mpn_add_nc(rp, rp, xp, n, xp[n]);
#else /* ! _nc */
cy = xp[n] + mpn_add_n(rp, rp, xp, n); /* xp[n] == 1 implies {xp,n} == ZERO */
#endif
cy += (rp[0]&1);
mpn_rshift(rp, rp, n, 1);
ASSERT (cy <= 2);
hi = (cy<<(GMP_NUMB_BITS-1))&GMP_NUMB_MASK; /* (cy&1) << ... */
cy >>= 1;
/* We can have cy != 0 only if hi = 0... */
ASSERT ((rp[n-1] & GMP_NUMB_HIGHBIT) == 0);
rp[n-1] |= hi;
/* ... rp[n-1] + cy can not overflow, the following INCR is correct. */
#endif
ASSERT (cy <= 1);
/* Next increment can not overflow, read the previous comments about cy. */
ASSERT ((cy == 0) || ((rp[n-1] & GMP_NUMB_HIGHBIT) == 0));
MPN_INCR_U(rp, n, cy);
/* Compute the highest half:
([(xp + xm)/2 mod (B^n-1)] - xp ) * B^n
*/
if (UNLIKELY (an + bn < rn))
{
/* Note that in this case, the only way the result can equal
zero mod B^{rn} - 1 is if one of the inputs is zero, and
then the output of both the recursive calls and this CRT
reconstruction is zero, not B^{rn} - 1. Which is good,
since the latter representation doesn't fit in the output
area.*/
cy = mpn_sub_n (rp + n, rp, xp, an + bn - n);
/* FIXME: This subtraction of the high parts is not really
necessary, we do it to get the carry out, and for sanity
checking. */
cy = xp[n] + mpn_sub_nc (xp + an + bn - n, rp + an + bn - n,
xp + an + bn - n, rn - (an + bn), cy);
ASSERT (an + bn == rn - 1 ||
mpn_zero_p (xp + an + bn - n + 1, rn - 1 - (an + bn)));
cy = mpn_sub_1 (rp, rp, an + bn, cy);
ASSERT (cy == (xp + an + bn - n)[0]);
}
else
{
cy = xp[n] + mpn_sub_n (rp + n, rp, xp, n);
/* cy = 1 only if {xp,n+1} is not ZERO, i.e. {rp,n} is not ZERO.
DECR will affect _at most_ the lowest n limbs. */
MPN_DECR_U (rp, 2*n, cy);
}
#undef a0
#undef a1
#undef b0
#undef b1
#undef xp
#undef sp1
}
}
/* put in {c, 2n} where n = 2k+r the value of {v0,2k} (already in place)
+ B^k * [{v1, 2k+1} - {t1, 2k+1}]
+ B^(2k) * [{t2, 2k+1} - {v0+vinf, 2k}]
+ B^(3k) * [{t1, 2k+1} - {t2, 2k+1}]
+ B^(4k) * {vinf,2r} (high 2r-1 limbs already in place)
where {t1, 2k+1} = (3*{v0,2k}+2*sa*{vm1,2k+1}+{v2,2k+1})/6-2*{vinf,2r}
{t2, 2k+1} = ({v1, 2k+1} + sa * {vm1, 2k+1})/2
(sa is the sign of {vm1, 2k+1}).
{vinf, 2r} stores the content of {v0, 2r} + {vinf, 2r}, with carry in cinf0.
vinf0 is the low limb of vinf.
ws is temporary space, and should have at least 2r limbs.
Think about:
The evaluated point a-b+c stands a good chance of having a zero carry
limb, a+b+c would have a 1/4 chance, and 4*a+2*b+c a 1/8 chance, roughly.
Perhaps this could be tested and stripped. Doing so before recursing
would be better than stripping at the start of mpn_toom3_mul_n/sqr_n,
since then the recursion could be based on the new size. Although in
truth the kara vs toom3 crossover is never so exact that one limb either
way makes a difference.
A small value like 1 or 2 for the carry could perhaps also be handled
with an add_n or addlsh1_n. Would that be faster than an extra limb on a
(recursed) multiply/square?
*/
static void
toom3_interpolate (mp_ptr c, mp_srcptr v1, mp_ptr v2, mp_ptr vm1,
mp_ptr vinf, mp_size_t k, mp_size_t r, int sa,
mp_limb_t vinf0, mp_limb_t cinf0, mp_ptr ws)
{
mp_limb_t cy, saved;
unsigned long twok = k + k;
unsigned long kk1 = twok + 1;
unsigned long twor = r + r;
mp_ptr c1, c2, c3, c4, c5;
mp_limb_t cout; /* final carry, should be zero at the end */
c1 = c + k;
c2 = c1 + k;
c3 = c2 + k;
c4 = c3 + k;
c5 = c4 + k;
#define v0 (c)
/* {c,2k} {c+2k,2k+1} {c+4k+1,2r-1} {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 |vm1| hi(vinf) v1 v2+2vm1 vinf
+lo(v0) */
ASSERT_NOCARRY (mpn_divexact_by3 (v2, v2, kk1)); /* v2 <- v2 / 3 */
#ifdef HAVE_NATIVE_mpn_rsh1add_n
mpn_rsh1add_n (v2, v2, v0, twok); /* v2 <- (lo(v2)+v0) / 2, exact */
cy = v2[twok] & 1; /* add high limb of v2 divided by 2 */
v2[twok] >>= 1;
MPN_INCR_U (v2 + twok - 1, 2, cy << (GMP_NUMB_BITS - 1));
#else
v2[twok] += mpn_add_n (v2, v2, v0, twok);
mpn_rshift (v2, v2, kk1, 1);
#endif
/* {c,2k} {c+2k,2k+1} {c+4k+1,2r-1} {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 |vm1| hi(vinf) v1 (3v0+2vm1+v2) vinf
/6 +lo(v0) */
/* vm1 <- t2 := (v1 + sa*vm1) / 2
t2 = a0*b0+a0*b2+a1*b1+a2*b0+a2*b2 >= 0
No carry comes out from {v1, kk1} +/- {vm1, kk1},
and the division by two is exact */
if (sa >= 0)
{
#ifdef HAVE_NATIVE_mpn_rsh1add_n
mpn_rsh1add_n (vm1, v1, vm1, kk1);
#else
mpn_add_n (vm1, vm1, v1, kk1);
mpn_rshift (vm1, vm1, kk1, 1);
#endif
}
else
{
#ifdef HAVE_NATIVE_mpn_rsh1sub_n
mpn_rsh1sub_n (vm1, v1, vm1, kk1);
#else
mpn_sub_n (vm1, v1, vm1, kk1);
mpn_rshift (vm1, vm1, kk1, 1);
#endif
}
/* {c,2k} {c+2k,2k+1} {c+4k+1,2r-1} {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 t2 hi(vinf) v1 t1 vinf+lo(v0) */
/* subtract 2*vinf to v2,
result is t1 := a0*b0+a0*b2+a1*b1+a1*b2+a2*b0+a2*b1+a2*b2 >= 0 */
saved = c4[0];
c4[0] = vinf0;
#ifdef HAVE_NATIVE_mpn_sublsh1_n
cy = mpn_sublsh1_n (v2, v2, c4, twor);
#else
cy = mpn_lshift (ws, c4, twor, 1);
cy += mpn_sub_n (v2, v2, ws, twor);
#endif
MPN_DECR_U (v2 + twor, kk1 - twor, cy);
c4[0] = saved;
/* subtract {t2, 2k+1} in {c+3k, 2k+1} i.e. in {t2+k, 2k+1}:
by chunks of k limbs from right to left to avoid overlap */
#define t2 (vm1)
/* a borrow may occur in one of the 2 following __GMPN_SUB_1 calls, but since
the final result is nonnegative, it will be compensated later on */
__GMPN_SUB_1 (cout, c5, c5, twor - k, t2[twok]);
cy = mpn_sub_n (c4, c4, t2 + k, k);
__GMPN_SUB_1 (cout, c5, c5, twor - k, cy);
cy = mpn_sub_n (c3, c3, t2, k);
__GMPN_SUB_1 (cout, c4, c4, twor, cy);
/* don't forget to add vinf0 in {c+4k, ...} */
__GMPN_ADD_1 (cout, c4, c4, twor, vinf0);
/* c c+k c+2k c+3k c+4k+1 t t+2k+1 t+4k+2
v0 t2 hi(vinf) v1 t1 vinf
-t2 +lo(v0)
*/
/* c c+k c+2k c+3k c+4k t t+2k+1 t+4k+2
v0 t2 vinf v1 t1 vinf
-t2 +lo(v0)
*/
/* subtract v0+vinf in {c+2k, ...} */
cy = cinf0 + mpn_sub_n (c2, c2, vinf, twor);
if (twor < twok)
{
__GMPN_SUB_1 (cy, c2 + twor, c2 + twor, twok - twor, cy);
cy += mpn_sub_n (c2 + twor, c2 + twor, v0 + twor, twok - twor);
}
__GMPN_SUB_1 (cout, c4, c4, twor, cy); /* 2n-4k = 2r */
/* c c+k c+2k c+3k c+4k t t+2k+1 t+4k+2
v0 t2 vinf v1 t1 vinf
-v0 -t2 +lo(v0)
-vinf */
/* subtract t1 in {c+k, ...} */
cy = mpn_sub_n (c1, c1, v2, kk1);
__GMPN_SUB_1 (cout, c3 + 1, c3 + 1, twor + k - 1, cy); /* 2n-(3k+1)=k+2r-1 */
/* c c+k c+2k c+3k c+4k t t+2k+1 t+4k+2
v0 t2 vinf v1 t1 vinf
-t1 -v0 -t2
-vinf */
/* add t1 in {c+3k, ...} */
cy = mpn_add_n (c3, c3, v2, kk1);
__GMPN_ADD_1 (cout, c5 + 1, c5 + 1, twor - k - 1, cy); /* 2n-(5k+1) = 2r-k-1 */
/* c c+k c+2k c+3k c+4k t t+2k+1 t+4k+2
v0 t2 t1 vinf v1 t1 vinf
-t1 -v0 -t2
-vinf */
/* add v1 in {c+k, ...} */
cy = mpn_add_n (c1, c1, v1, kk1);
__GMPN_ADD_1 (cout, c3 + 1, c3 + 1, twor + k - 1, cy); /* 2n-(3k+1) = 2r+k-1 */
/* c c+k c+2k c+3k c+4k t t+2k+1 t+4k+2
v0 v1 t2 t1 vinf v1 t1 vinf
-t1 -v0 -t2
-vinf */
#undef v0
#undef t2
}
void
mpn_toom_interpolate_5pts (mp_ptr c, mp_ptr v2, mp_ptr vm1,
mp_size_t k, mp_size_t twor, int sa,
mp_limb_t vinf0)
{
mp_limb_t cy, saved;
mp_size_t twok;
mp_size_t kk1;
mp_ptr c1, v1, c3, vinf;
twok = k + k;
kk1 = twok + 1;
c1 = c + k;
v1 = c1 + k;
c3 = v1 + k;
vinf = c3 + k;
#define v0 (c)
/* (1) v2 <- v2-vm1 < v2+|vm1|, (16 8 4 2 1) - (1 -1 1 -1 1) =
thus 0 <= v2 < 50*B^(2k) < 2^6*B^(2k) (15 9 3 3 0)
*/
if (sa)
ASSERT_NOCARRY (mpn_add_n (v2, v2, vm1, kk1));
else
ASSERT_NOCARRY (mpn_sub_n (v2, v2, vm1, kk1));
/* {c,2k} {c+2k,2k+1} {c+4k+1,2r-1} {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 v1 hi(vinf) |vm1| v2-vm1 EMPTY */
ASSERT_NOCARRY (mpn_divexact_by3 (v2, v2, kk1)); /* v2 <- v2 / 3 */
/* (5 3 1 1 0)*/
/* {c,2k} {c+2k,2k+1} {c+4k+1,2r-1} {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 v1 hi(vinf) |vm1| (v2-vm1)/3 EMPTY */
/* (2) vm1 <- tm1 := (v1 - vm1) / 2 [(1 1 1 1 1) - (1 -1 1 -1 1)] / 2 =
tm1 >= 0 (0 1 0 1 0)
No carry comes out from {v1, kk1} +/- {vm1, kk1},
and the division by two is exact.
If (sa!=0) the sign of vm1 is negative */
if (sa)
{
#ifdef HAVE_NATIVE_mpn_rsh1add_n
mpn_rsh1add_n (vm1, v1, vm1, kk1);
#else
ASSERT_NOCARRY (mpn_add_n (vm1, v1, vm1, kk1));
ASSERT_NOCARRY (mpn_rshift (vm1, vm1, kk1, 1));
#endif
}
else
{
#ifdef HAVE_NATIVE_mpn_rsh1sub_n
mpn_rsh1sub_n (vm1, v1, vm1, kk1);
#else
ASSERT_NOCARRY (mpn_sub_n (vm1, v1, vm1, kk1));
ASSERT_NOCARRY (mpn_rshift (vm1, vm1, kk1, 1));
#endif
}
/* {c,2k} {c+2k,2k+1} {c+4k+1,2r-1} {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 v1 hi(vinf) tm1 (v2-vm1)/3 EMPTY */
/* (3) v1 <- t1 := v1 - v0 (1 1 1 1 1) - (0 0 0 0 1) = (1 1 1 1 0)
t1 >= 0
*/
vinf[0] -= mpn_sub_n (v1, v1, c, twok);
/* {c,2k} {c+2k,2k+1} {c+4k+1,2r-1} {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 v1-v0 hi(vinf) tm1 (v2-vm1)/3 EMPTY */
/* (4) v2 <- t2 := ((v2-vm1)/3-t1)/2 = (v2-vm1-3*t1)/6
t2 >= 0 [(5 3 1 1 0) - (1 1 1 1 0)]/2 = (2 1 0 0 0)
*/
#ifdef HAVE_NATIVE_mpn_rsh1sub_n
mpn_rsh1sub_n (v2, v2, v1, kk1);
#else
ASSERT_NOCARRY (mpn_sub_n (v2, v2, v1, kk1));
ASSERT_NOCARRY (mpn_rshift (v2, v2, kk1, 1));
#endif
/* {c,2k} {c+2k,2k+1} {c+4k+1,2r-1} {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 v1-v0 hi(vinf) tm1 (v2-vm1-3t1)/6 EMPTY */
/* (5) v1 <- t1-tm1 (1 1 1 1 0) - (0 1 0 1 0) = (1 0 1 0 0)
result is v1 >= 0
*/
ASSERT_NOCARRY (mpn_sub_n (v1, v1, vm1, kk1));
/* We do not need to read the value in vm1, so we add it in {c+k, ...} */
cy = mpn_add_n (c1, c1, vm1, kk1);
MPN_INCR_U (c3 + 1, twor + k - 1, cy); /* 2n-(3k+1) = 2r+k-1 */
/* Memory allocated for vm1 is now free, it can be recycled ...*/
/* (6) v2 <- v2 - 2*vinf, (2 1 0 0 0) - 2*(1 0 0 0 0) = (0 1 0 0 0)
result is v2 >= 0 */
saved = vinf[0]; /* Remember v1's highest byte (will be overwritten). */
vinf[0] = vinf0; /* Set the right value for vinf0 */
#ifdef HAVE_NATIVE_mpn_sublsh1_n
cy = mpn_sublsh1_n (v2, v2, vinf, twor);
#else
/* Overwrite unused vm1 */
cy = mpn_lshift (vm1, vinf, twor, 1);
cy += mpn_sub_n (v2, v2, vm1, twor);
#endif
MPN_DECR_U (v2 + twor, kk1 - twor, cy);
/* Current matrix is
[1 0 0 0 0; vinf
0 1 0 0 0; v2
1 0 1 0 0; v1
0 1 0 1 0; vm1
0 0 0 0 1] v0
Some vaues already are in-place (we added vm1 in the correct position)
| vinf| v1 | v0 |
| vm1 |
One still is in a separated area
| +v2 |
We have to compute v1-=vinf; vm1 -= v2,
|-vinf|
| -v2 |
Carefully reordering operations we can avoid to compute twice the sum
of the high half of v2 plus the low half of vinf.
*/
/* Add the high half of t2 in {vinf} */
if ( LIKELY(twor > k + 1) ) { /* This is the expected flow */
cy = mpn_add_n (vinf, vinf, v2 + k, k + 1);
MPN_INCR_U (c3 + kk1, twor - k - 1, cy); /* 2n-(5k+1) = 2r-k-1 */
} else { /* triggered only by very unbalanced cases like
(k+k+(k-2))x(k+k+1) , should be handled by toom32 */
ASSERT_NOCARRY (mpn_add_n (vinf, vinf, v2 + k, twor));
}
/* (7) v1 <- v1 - vinf, (1 0 1 0 0) - (1 0 0 0 0) = (0 0 1 0 0)
result is >= 0 */
/* Side effect: we also subtracted (high half) vm1 -= v2 */
cy = mpn_sub_n (v1, v1, vinf, twor); /* vinf is at most twor long. */
vinf0 = vinf[0]; /* Save again the right value for vinf0 */
vinf[0] = saved;
MPN_DECR_U (v1 + twor, kk1 - twor, cy); /* Treat the last bytes. */
/* (8) vm1 <- vm1-v2 (0 1 0 1 0) - (0 1 0 0 0) = (0 0 0 1 0)
Operate only on the low half.
*/
cy = mpn_sub_n (c1, c1, v2, k);
MPN_DECR_U (v1, kk1, cy);
/********************* Beginning the final phase **********************/
/* Most of the recomposition was done */
/* add t2 in {c+3k, ...}, but only the low half */
cy = mpn_add_n (c3, c3, v2, k);
vinf[0] += cy;
ASSERT(vinf[0] >= cy); /* No carry */
MPN_INCR_U (vinf, twor, vinf0); /* Add vinf0, propagate carry. */
#undef v0
}
void
mpn_toom_interpolate_7pts (mp_ptr rp, mp_size_t n, enum toom7_flags flags,
mp_ptr w1, mp_ptr w3, mp_ptr w4, mp_ptr w5,
mp_size_t w6n, mp_ptr tp)
{
mp_size_t m;
mp_limb_t cy;
m = 2*n + 1;
#define w0 rp
#define w2 (rp + 2*n)
#define w6 (rp + 6*n)
ASSERT (w6n > 0);
ASSERT (w6n <= 2*n);
/* Using formulas similar to Marco Bodrato's
W5 = W5 + W4
W1 =(W4 - W1)/2
W4 = W4 - W0
W4 =(W4 - W1)/4 - W6*16
W3 =(W2 - W3)/2
W2 = W2 - W3
W5 = W5 - W2*65 May be negative.
W2 = W2 - W6 - W0
W5 =(W5 + W2*45)/2 Now >= 0 again.
W4 =(W4 - W2)/3
W2 = W2 - W4
W1 = W5 - W1 May be negative.
W5 =(W5 - W3*8)/9
W3 = W3 - W5
W1 =(W1/15 + W5)/2 Now >= 0 again.
W5 = W5 - W1
where W0 = f(0), W1 = f(-2), W2 = f(1), W3 = f(-1),
W4 = f(2), W5 = f(1/2), W6 = f(oo),
Note that most intermediate results are positive; the ones that
may be negative are represented in two's complement. We must
never shift right a value that may be negative, since that would
invalidate the sign bit. On the other hand, divexact by odd
numbers work fine with two's complement.
*/
mpn_add_n (w5, w5, w4, m);
if (flags & toom7_w1_neg)
{
#ifdef HAVE_NATIVE_mpn_rsh1add_n
mpn_rsh1add_n (w1, w1, w4, m);
#else
mpn_add_n (w1, w1, w4, m); ASSERT (!(w1[0] & 1));
mpn_rshift (w1, w1, m, 1);
#endif
}
else
{
#ifdef HAVE_NATIVE_mpn_rsh1sub_n
mpn_rsh1sub_n (w1, w4, w1, m);
#else
mpn_sub_n (w1, w4, w1, m); ASSERT (!(w1[0] & 1));
mpn_rshift (w1, w1, m, 1);
#endif
}
mpn_sub (w4, w4, m, w0, 2*n);
mpn_sub_n (w4, w4, w1, m); ASSERT (!(w4[0] & 3));
mpn_rshift (w4, w4, m, 2); /* w4>=0 */
tp[w6n] = mpn_lshift (tp, w6, w6n, 4);
mpn_sub (w4, w4, m, tp, w6n+1);
if (flags & toom7_w3_neg)
{
#ifdef HAVE_NATIVE_mpn_rsh1add_n
mpn_rsh1add_n (w3, w3, w2, m);
#else
mpn_add_n (w3, w3, w2, m); ASSERT (!(w3[0] & 1));
mpn_rshift (w3, w3, m, 1);
#endif
}
else
{
#ifdef HAVE_NATIVE_mpn_rsh1sub_n
mpn_rsh1sub_n (w3, w2, w3, m);
#else
mpn_sub_n (w3, w2, w3, m); ASSERT (!(w3[0] & 1));
mpn_rshift (w3, w3, m, 1);
#endif
}
mpn_sub_n (w2, w2, w3, m);
mpn_submul_1 (w5, w2, m, 65);
mpn_sub (w2, w2, m, w6, w6n);
mpn_sub (w2, w2, m, w0, 2*n);
mpn_addmul_1 (w5, w2, m, 45); ASSERT (!(w5[0] & 1));
mpn_rshift (w5, w5, m, 1);
mpn_sub_n (w4, w4, w2, m);
mpn_divexact_by3 (w4, w4, m);
mpn_sub_n (w2, w2, w4, m);
mpn_sub_n (w1, w5, w1, m);
mpn_lshift (tp, w3, m, 3);
mpn_sub_n (w5, w5, tp, m);
mpn_divexact_by9 (w5, w5, m);
mpn_sub_n (w3, w3, w5, m);
mpn_divexact_by15 (w1, w1, m);
mpn_add_n (w1, w1, w5, m); ASSERT (!(w1[0] & 1));
mpn_rshift (w1, w1, m, 1); /* w1>=0 now */
mpn_sub_n (w5, w5, w1, m);
/* These bounds are valid for the 4x4 polynomial product of toom44,
* and they are conservative for toom53 and toom62. */
ASSERT (w1[2*n] < 2);
ASSERT (w2[2*n] < 3);
ASSERT (w3[2*n] < 4);
ASSERT (w4[2*n] < 3);
ASSERT (w5[2*n] < 2);
/* Addition chain. Note carries and the 2n'th limbs that need to be
* added in.
*
* Special care is needed for w2[2n] and the corresponding carry,
* since the "simple" way of adding it all together would overwrite
* the limb at wp[2*n] and rp[4*n] (same location) with the sum of
* the high half of w3 and the low half of w4.
*
* 7 6 5 4 3 2 1 0
* | | | | | | | | |
* ||w3 (2n+1)|
* ||w4 (2n+1)|
* ||w5 (2n+1)| ||w1 (2n+1)|
* + | w6 (w6n)| ||w2 (2n+1)| w0 (2n) | (share storage with r)
* -----------------------------------------------
* r | | | | | | | | |
* c7 c6 c5 c4 c3 Carries to propagate
*/
cy = mpn_add_n (rp + n, rp + n, w1, m);
MPN_INCR_U (w2 + n + 1, n , cy);
cy = mpn_add_n (rp + 3*n, rp + 3*n, w3, n);
MPN_INCR_U (w3 + n, n + 1, w2[2*n] + cy);
cy = mpn_add_n (rp + 4*n, w3 + n, w4, n);
MPN_INCR_U (w4 + n, n + 1, w3[2*n] + cy);
cy = mpn_add_n (rp + 5*n, w4 + n, w5, n);
MPN_INCR_U (w5 + n, n + 1, w4[2*n] + cy);
if (w6n > n + 1)
ASSERT_NOCARRY (mpn_add (rp + 6*n, rp + 6*n, w6n, w5 + n, n + 1));
else
{
ASSERT_NOCARRY (mpn_add_n (rp + 6*n, rp + 6*n, w5 + n, w6n));
#if WANT_ASSERT
{
mp_size_t i;
for (i = w6n; i <= n; i++)
ASSERT (w5[n + i] == 0);
}
#endif
}
}
void
mpn_toom_interpolate_5pts (mp_ptr c, mp_ptr v2, mp_ptr vm1,
mp_size_t k, mp_size_t twor, int sa,
mp_limb_t vinf0, mp_ptr ws)
{
mp_limb_t cy, saved;
mp_size_t twok = k + k;
mp_size_t kk1 = twok + 1;
mp_ptr c1, v1, c3, vinf, c5;
mp_limb_t cout; /* final carry, should be zero at the end */
c1 = c + k;
v1 = c1 + k;
c3 = v1 + k;
vinf = c3 + k;
c5 = vinf + k;
#define v0 (c)
/* (1) v2 <- v2-vm1 < v2+|vm1|, (16 8 4 2 1) - (1 -1 1 -1 1) =
thus 0 <= v2 < 50*B^(2k) < 2^6*B^(2k) (15 9 3 3 0)
*/
if (sa <= 0)
mpn_add_n (v2, v2, vm1, kk1);
else
mpn_sub_n (v2, v2, vm1, kk1);
/* {c,2k} {c+2k,2k+1} {c+4k+1,2r-1} {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 v1 hi(vinf) |vm1| v2-vm1 EMPTY */
ASSERT_NOCARRY (mpn_divexact_by3 (v2, v2, kk1)); /* v2 <- v2 / 3 */
/* (5 3 1 1 0)*/
/* {c,2k} {c+2k,2k+1} {c+4k+1,2r-1} {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 v1 hi(vinf) |vm1| (v2-vm1)/3 EMPTY */
/* (2) vm1 <- tm1 := (v1 - sa*vm1) / 2 [(1 1 1 1 1) - (1 -1 1 -1 1)] / 2 =
tm1 >= 0 (0 1 0 1 0)
No carry comes out from {v1, kk1} +/- {vm1, kk1},
and the division by two is exact */
if (sa <= 0)
{
#ifdef HAVE_NATIVE_mpn_rsh1add_n
mpn_rsh1add_n (vm1, v1, vm1, kk1);
#else
mpn_add_n (vm1, v1, vm1, kk1);
mpn_rshift (vm1, vm1, kk1, 1);
#endif
}
else
{
#ifdef HAVE_NATIVE_mpn_rsh1sub_n
mpn_rsh1sub_n (vm1, v1, vm1, kk1);
#else
mpn_sub_n (vm1, v1, vm1, kk1);
mpn_rshift (vm1, vm1, kk1, 1);
#endif
}
/* {c,2k} {c+2k,2k+1} {c+4k+1,2r-1} {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 v1 hi(vinf) tm1 (v2-vm1)/3 EMPTY */
/* (3) v1 <- t1 := v1 - v0 (1 1 1 1 1) - (0 0 0 0 1) = (1 1 1 1 0)
t1 >= 0
*/
vinf[0] -= mpn_sub_n (v1, v1, c, twok);
/* {c,2k} {c+2k,2k+1} {c+4k+1,2r-1} {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 v1-v0 hi(vinf) tm1 (v2-vm1)/3 EMPTY */
/* (4) v2 <- t2 := ((v2-vm1)/3-t1)/2 = (v2-vm1-3*t1)/6
t2 >= 0 [(5 3 1 1 0) - (1 1 1 1 0)]/2 = (2 1 0 0 0)
*/
#ifdef HAVE_NATIVE_mpn_rsh1sub_n
mpn_rsh1sub_n (v2, v2, v1, kk1);
#else
mpn_sub_n (v2, v2, v1, kk1);
mpn_rshift (v2, v2, kk1, 1);
#endif
/* {c,2k} {c+2k,2k+1} {c+4k+1,2r-1} {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 v1-v0 hi(vinf) tm1 (v2-vm1-3t1)/6 EMPTY */
/* (5) v1 <- t1-tm1 (1 1 1 1 0) - (0 1 0 1 0) = (1 0 1 0 0)
result is v1 >= 0
*/
mpn_sub_n (v1, v1, vm1, kk1);
/* {c,2k} {c+2k,2k+1} {c+4k+1,2r-1} {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 v1-v0-tm1 hi(vinf) tm1 (v2-vm1-3t1)/6 EMPTY */
/* (6) v2 <- v2 - 2*vinf, (2 1 0 0 0) - 2*(1 0 0 0 0) = (0 1 0 0 0)
result is v2 >= 0 */
saved = vinf[0]; /* Remember v1's highest byte (will be overwritten). */
vinf[0] = vinf0; /* Set the right value for vinf0 */
#ifdef HAVE_NATIVE_mpn_sublsh1_n
cy = mpn_sublsh1_n (v2, v2, vinf, twor);
#else
cy = mpn_lshift (ws, vinf, twor, 1);
cy += mpn_sub_n (v2, v2, ws, twor);
#endif
MPN_DECR_U (v2 + twor, kk1 - twor, cy);
/* (7) v1 <- v1 - vinf, (1 0 1 0 0) - (1 0 0 0 0) = (0 0 1 0 0)
result is >= 0 */
cy = mpn_sub_n (v1, v1, vinf, twor); /* vinf is at most twor long. */
vinf[0] = saved;
MPN_DECR_U (v1 + twor, kk1 - twor, cy); /* Treat the last bytes. */
__GMPN_ADD_1 (cout, vinf, vinf, twor, vinf0); /* Add vinf0, propagate carry. */
/* (8) vm1 <- vm1-t2 (0 1 0 1 0) - (0 1 0 0 0) = (0 0 0 1 0)
vm1 >= 0
*/
mpn_sub_n (vm1, vm1, v2, kk1); /* No overlapping here. */
/********************* Beginning the final phase **********************/
/* {c,2k} {c+2k,2k } {c+4k ,2r } {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 t1 hi(t1)+vinf tm1 (v2-vm1-3t1)/6 EMPTY */
/* (9) add t2 in {c+3k, ...} */
cy = mpn_add_n (c3, c3, v2, kk1);
__GMPN_ADD_1 (cout, c5 + 1, c5 + 1, twor - k - 1, cy); /* 2n-(5k+1) = 2r-k-1 */
/* {c,2k} {c+2k,2k } {c+4k ,2r } {t,2k+1} {t+2k+1,2k+1} {t+4k+2,2r}
v0 t1 hi(t1)+vinf tm1 (v2-vm1-3t1)/6 EMPTY */
/* c c+k c+2k c+3k c+4k t t+2k+1 t+4k+2
v0 t1 vinf tm1 t2
+t2 */
/* add vm1 in {c+k, ...} */
cy = mpn_add_n (c1, c1, vm1, kk1);
__GMPN_ADD_1 (cout, c3 + 1, c3 + 1, twor + k - 1, cy); /* 2n-(3k+1) = 2r+k-1 */
/* c c+k c+2k c+3k c+4k t t+2k+1 t+4k+2
v0 t1 vinf tm1 t2
+tm1 +t2 */
#undef v0
#undef t2
}
void
mpn_toom_interpolate_6pts (mp_ptr pp, mp_size_t n, enum toom6_flags flags,
mp_ptr w4, mp_ptr w2, mp_ptr w1,
mp_size_t w0n)
{
mp_limb_t cy;
/* cy6 can be stored in w1[2*n], cy4 in w4[0], embankment in w2[0] */
mp_limb_t cy4, cy6, embankment;
ASSERT( n > 0 );
ASSERT( 2*n >= w0n && w0n > 0 );
#define w5 pp /* 2n */
#define w3 (pp + 2 * n) /* 2n+1 */
#define w0 (pp + 5 * n) /* w0n */
/* Interpolate with sequence:
W2 =(W1 - W2)>>2
W1 =(W1 - W5)>>1
W1 =(W1 - W2)>>1
W4 =(W3 - W4)>>1
W2 =(W2 - W4)/3
W3 = W3 - W4 - W5
W1 =(W1 - W3)/3
// Last steps are mixed with recomposition...
W2 = W2 - W0<<2
W4 = W4 - W2
W3 = W3 - W1
W2 = W2 - W0
*/
/* W2 =(W1 - W2)>>2 */
if (flags & toom6_vm2_neg)
mpn_add_n (w2, w1, w2, 2 * n + 1);
else
mpn_sub_n (w2, w1, w2, 2 * n + 1);
mpn_rshift (w2, w2, 2 * n + 1, 2);
/* W1 =(W1 - W5)>>1 */
w1[2*n] -= mpn_sub_n (w1, w1, w5, 2*n);
mpn_rshift (w1, w1, 2 * n + 1, 1);
/* W1 =(W1 - W2)>>1 */
#if HAVE_NATIVE_mpn_rsh1sub_n
mpn_rsh1sub_n (w1, w1, w2, 2 * n + 1);
#else
mpn_sub_n (w1, w1, w2, 2 * n + 1);
mpn_rshift (w1, w1, 2 * n + 1, 1);
#endif
/* W4 =(W3 - W4)>>1 */
if (flags & toom6_vm1_neg)
{
#if HAVE_NATIVE_mpn_rsh1add_n
mpn_rsh1add_n (w4, w3, w4, 2 * n + 1);
#else
mpn_add_n (w4, w3, w4, 2 * n + 1);
mpn_rshift (w4, w4, 2 * n + 1, 1);
#endif
}
else
{
#if HAVE_NATIVE_mpn_rsh1sub_n
mpn_rsh1sub_n (w4, w3, w4, 2 * n + 1);
#else
mpn_sub_n (w4, w3, w4, 2 * n + 1);
mpn_rshift (w4, w4, 2 * n + 1, 1);
#endif
}
/* W2 =(W2 - W4)/3 */
mpn_sub_n (w2, w2, w4, 2 * n + 1);
mpn_divexact_by3 (w2, w2, 2 * n + 1);
/* W3 = W3 - W4 - W5 */
mpn_sub_n (w3, w3, w4, 2 * n + 1);
w3[2 * n] -= mpn_sub_n (w3, w3, w5, 2 * n);
/* W1 =(W1 - W3)/3 */
mpn_sub_n (w1, w1, w3, 2 * n + 1);
mpn_divexact_by3 (w1, w1, 2 * n + 1);
/*
[1 0 0 0 0 0;
0 1 0 0 0 0;
1 0 1 0 0 0;
0 1 0 1 0 0;
1 0 1 0 1 0;
0 0 0 0 0 1]
pp[] prior to operations:
|_H w0__|_L w0__|______||_H w3__|_L w3__|_H w5__|_L w5__|
summation scheme for remaining operations:
|______________5|n_____4|n_____3|n_____2|n______|n______|pp
|_H w0__|_L w0__|______||_H w3__|_L w3__|_H w5__|_L w5__|
|| H w4 | L w4 |
|| H w2 | L w2 |
|| H w1 | L w1 |
||-H w1 |-L w1 |
|-H w0 |-L w0 ||-H w2 |-L w2 |
*/
cy = mpn_add_n (pp + n, pp + n, w4, 2 * n + 1);
MPN_INCR_U (pp + 3 * n + 1, n, cy);
/* W2 -= W0<<2 */
#if HAVE_NATIVE_mpn_sublsh_n || HAVE_NATIVE_mpn_sublsh2_n_ip1
#if HAVE_NATIVE_mpn_sublsh2_n_ip1
cy = mpn_sublsh2_n_ip1 (w2, w0, w0n);
#else
cy = mpn_sublsh_n (w2, w2, w0, w0n, 2);
#endif
#else
/* {W4,2*n+1} is now free and can be overwritten. */
cy = mpn_lshift(w4, w0, w0n, 2);
cy+= mpn_sub_n(w2, w2, w4, w0n);
#endif
MPN_DECR_U (w2 + w0n, 2 * n + 1 - w0n, cy);
/* W4L = W4L - W2L */
cy = mpn_sub_n (pp + n, pp + n, w2, n);
MPN_DECR_U (w3, 2 * n + 1, cy);
/* W3H = W3H + W2L */
cy4 = w3[2 * n] + mpn_add_n (pp + 3 * n, pp + 3 * n, w2, n);
/* W1L + W2H */
cy = w2[2 * n] + mpn_add_n (pp + 4 * n, w1, w2 + n, n);
MPN_INCR_U (w1 + n, n + 1, cy);
/* W0 = W0 + W1H */
if (LIKELY (w0n > n))
cy6 = w1[2 * n] + mpn_add_n (w0, w0, w1 + n, n);
else
cy6 = mpn_add_n (w0, w0, w1 + n, w0n);
/*
summation scheme for the next operation:
|...____5|n_____4|n_____3|n_____2|n______|n______|pp
|...w0___|_w1_w2_|_H w3__|_L w3__|_H w5__|_L w5__|
...-w0___|-w1_w2 |
*/
/* if(LIKELY(w0n>n)) the two operands below DO overlap! */
cy = mpn_sub_n (pp + 2 * n, pp + 2 * n, pp + 4 * n, n + w0n);
/* embankment is a "dirty trick" to avoid carry/borrow propagation
beyond allocated memory */
embankment = w0[w0n - 1] - 1;
w0[w0n - 1] = 1;
if (LIKELY (w0n > n)) {
if (cy4 > cy6)
MPN_INCR_U (pp + 4 * n, w0n + n, cy4 - cy6);
else
MPN_DECR_U (pp + 4 * n, w0n + n, cy6 - cy4);
MPN_DECR_U (pp + 3 * n + w0n, 2 * n, cy);
MPN_INCR_U (w0 + n, w0n - n, cy6);
} else {
MPN_INCR_U (pp + 4 * n, w0n + n, cy4);
MPN_DECR_U (pp + 3 * n + w0n, 2 * n, cy + cy6);
}
w0[w0n - 1] += embankment;
#undef w5
#undef w3
#undef w0
}
void
check (void)
{
mp_limb_t wp[100], xp[100], yp[100];
mp_size_t size = 100;
refmpn_zero (xp, size);
refmpn_zero (yp, size);
refmpn_zero (wp, size);
pre ("mpn_add_n");
mpn_add_n (wp, xp, yp, size);
post ();
#if HAVE_NATIVE_mpn_add_nc
pre ("mpn_add_nc");
mpn_add_nc (wp, xp, yp, size, CNST_LIMB(0));
post ();
#endif
#if HAVE_NATIVE_mpn_addlsh1_n
pre ("mpn_addlsh1_n");
mpn_addlsh1_n (wp, xp, yp, size);
post ();
#endif
#if HAVE_NATIVE_mpn_and_n
pre ("mpn_and_n");
mpn_and_n (wp, xp, yp, size);
post ();
#endif
#if HAVE_NATIVE_mpn_andn_n
pre ("mpn_andn_n");
mpn_andn_n (wp, xp, yp, size);
post ();
#endif
pre ("mpn_addmul_1");
mpn_addmul_1 (wp, xp, size, yp[0]);
post ();
#if HAVE_NATIVE_mpn_addmul_1c
pre ("mpn_addmul_1c");
mpn_addmul_1c (wp, xp, size, yp[0], CNST_LIMB(0));
post ();
#endif
#if HAVE_NATIVE_mpn_com_n
pre ("mpn_com_n");
mpn_com_n (wp, xp, size);
post ();
#endif
#if HAVE_NATIVE_mpn_copyd
pre ("mpn_copyd");
mpn_copyd (wp, xp, size);
post ();
#endif
#if HAVE_NATIVE_mpn_copyi
pre ("mpn_copyi");
mpn_copyi (wp, xp, size);
post ();
#endif
pre ("mpn_divexact_1");
mpn_divexact_1 (wp, xp, size, CNST_LIMB(123));
post ();
pre ("mpn_divexact_by3c");
mpn_divexact_by3c (wp, xp, size, CNST_LIMB(0));
post ();
pre ("mpn_divrem_1");
mpn_divrem_1 (wp, (mp_size_t) 0, xp, size, CNST_LIMB(123));
post ();
#if HAVE_NATIVE_mpn_divrem_1c
pre ("mpn_divrem_1c");
mpn_divrem_1c (wp, (mp_size_t) 0, xp, size, CNST_LIMB(123), CNST_LIMB(122));
post ();
#endif
pre ("mpn_gcd_1");
xp[0] |= 1;
notdead += (unsigned long) mpn_gcd_1 (xp, size, CNST_LIMB(123));
post ();
#if HAVE_NATIVE_mpn_gcd_finda
pre ("mpn_gcd_finda");
xp[0] |= 1;
xp[1] |= 1;
notdead += mpn_gcd_finda (xp);
post ();
#endif
pre ("mpn_hamdist");
notdead += mpn_hamdist (xp, yp, size);
post ();
#if HAVE_NATIVE_mpn_ior_n
pre ("mpn_ior_n");
mpn_ior_n (wp, xp, yp, size);
post ();
#endif
#if HAVE_NATIVE_mpn_iorn_n
pre ("mpn_iorn_n");
mpn_iorn_n (wp, xp, yp, size);
post ();
#endif
pre ("mpn_lshift");
mpn_lshift (wp, xp, size, 1);
post ();
pre ("mpn_mod_1");
notdead += mpn_mod_1 (xp, size, CNST_LIMB(123));
post ();
#if HAVE_NATIVE_mpn_mod_1c
pre ("mpn_mod_1c");
notdead += mpn_mod_1c (xp, size, CNST_LIMB(123), CNST_LIMB(122));
post ();
#endif
#if GMP_NUMB_BITS % 4 == 0
pre ("mpn_mod_34lsub1");
notdead += mpn_mod_34lsub1 (xp, size);
post ();
#endif
pre ("mpn_modexact_1_odd");
notdead += mpn_modexact_1_odd (xp, size, CNST_LIMB(123));
post ();
pre ("mpn_modexact_1c_odd");
notdead += mpn_modexact_1c_odd (xp, size, CNST_LIMB(123), CNST_LIMB(456));
post ();
pre ("mpn_mul_1");
mpn_mul_1 (wp, xp, size, yp[0]);
post ();
#if HAVE_NATIVE_mpn_mul_1c
pre ("mpn_mul_1c");
mpn_mul_1c (wp, xp, size, yp[0], CNST_LIMB(0));
post ();
#endif
#if HAVE_NATIVE_mpn_mul_2
pre ("mpn_mul_2");
mpn_mul_2 (wp, xp, size-1, yp);
post ();
#endif
pre ("mpn_mul_basecase");
mpn_mul_basecase (wp, xp, (mp_size_t) 3, yp, (mp_size_t) 3);
post ();
#if HAVE_NATIVE_mpn_nand_n
pre ("mpn_nand_n");
mpn_nand_n (wp, xp, yp, size);
post ();
#endif
#if HAVE_NATIVE_mpn_nior_n
pre ("mpn_nior_n");
mpn_nior_n (wp, xp, yp, size);
post ();
#endif
pre ("mpn_popcount");
notdead += mpn_popcount (xp, size);
post ();
pre ("mpn_preinv_mod_1");
notdead += mpn_preinv_mod_1 (xp, size, GMP_NUMB_MAX,
refmpn_invert_limb (GMP_NUMB_MAX));
post ();
#if USE_PREINV_DIVREM_1 || HAVE_NATIVE_mpn_preinv_divrem_1
pre ("mpn_preinv_divrem_1");
mpn_preinv_divrem_1 (wp, (mp_size_t) 0, xp, size, GMP_NUMB_MAX,
refmpn_invert_limb (GMP_NUMB_MAX), 0);
post ();
#endif
#if HAVE_NATIVE_mpn_rsh1add_n
pre ("mpn_rsh1add_n");
mpn_rsh1add_n (wp, xp, yp, size);
post ();
#endif
#if HAVE_NATIVE_mpn_rsh1sub_n
pre ("mpn_rsh1sub_n");
mpn_rsh1sub_n (wp, xp, yp, size);
post ();
#endif
pre ("mpn_rshift");
mpn_rshift (wp, xp, size, 1);
post ();
pre ("mpn_sqr_basecase");
mpn_sqr_basecase (wp, xp, (mp_size_t) 3);
post ();
pre ("mpn_submul_1");
mpn_submul_1 (wp, xp, size, yp[0]);
post ();
#if HAVE_NATIVE_mpn_submul_1c
pre ("mpn_submul_1c");
mpn_submul_1c (wp, xp, size, yp[0], CNST_LIMB(0));
post ();
#endif
pre ("mpn_sub_n");
mpn_sub_n (wp, xp, yp, size);
post ();
#if HAVE_NATIVE_mpn_sub_nc
pre ("mpn_sub_nc");
mpn_sub_nc (wp, xp, yp, size, CNST_LIMB(0));
post ();
#endif
#if HAVE_NATIVE_mpn_sublsh1_n
pre ("mpn_sublsh1_n");
mpn_sublsh1_n (wp, xp, yp, size);
post ();
#endif
#if HAVE_NATIVE_mpn_udiv_qrnnd
pre ("mpn_udiv_qrnnd");
mpn_udiv_qrnnd (&wp[0], CNST_LIMB(122), xp[0], CNST_LIMB(123));
post ();
#endif
#if HAVE_NATIVE_mpn_udiv_qrnnd_r
pre ("mpn_udiv_qrnnd_r");
mpn_udiv_qrnnd (CNST_LIMB(122), xp[0], CNST_LIMB(123), &wp[0]);
post ();
#endif
#if HAVE_NATIVE_mpn_umul_ppmm
pre ("mpn_umul_ppmm");
mpn_umul_ppmm (&wp[0], xp[0], yp[0]);
post ();
#endif
#if HAVE_NATIVE_mpn_umul_ppmm_r
pre ("mpn_umul_ppmm_r");
mpn_umul_ppmm_r (&wp[0], xp[0], yp[0]);
post ();
#endif
#if HAVE_NATIVE_mpn_xor_n
pre ("mpn_xor_n");
mpn_xor_n (wp, xp, yp, size);
post ();
#endif
#if HAVE_NATIVE_mpn_xnor_n
pre ("mpn_xnor_n");
mpn_xnor_n (wp, xp, yp, size);
post ();
#endif
}
/*
We have
{v0,2k} {v1,2k+1} {c+4k+1,r+r2-1}
v0 v1 {-}vinf
vinf0 is the first limb of vinf, which is overwritten by v1
{vm1,2k+1} {v2, 2k+1}
ws is temporary space
sa is the sign of vm1
rr2 is r+r2
We want to compute
t1 <- (3*v0+2*vm1+v2)/6-2*vinf
t2 <- (v1+vm1)/2
then the result is c0+c1*t+c2*t^2+c3*t^3+c4*t^4 where
c0 <- v0
c1 <- v1 - t1
c2 <- t2 - v0 - vinf
c3 <- t1 - t2
c4 <- vinf
*/
void
mpn_toom3_interpolate (mp_ptr c, mp_ptr v1, mp_ptr v2, mp_ptr vm1,
mp_ptr vinf, mp_size_t k, mp_size_t rr2, int sa,
mp_limb_t vinf0, mp_ptr ws)
{
mp_limb_t cy, saved;
mp_size_t twok = k + k;
mp_size_t kk1 = twok + 1;
mp_ptr c1, c2, c3, c4, c5;
mp_limb_t cout; /* final carry, should be zero at the end */
c1 = c + k;
c2 = c1 + k;
c3 = c2 + k;
c4 = c3 + k;
c5 = c4 + k;
#define v0 (c)
/* {c,2k} {c+2k,2k+1} {c+4k+1,r+r2-1}
v0 v1 {-}vinf
{vm1,2k+1} {v2, 2k+1}
*/
/* v2 <- v2 - vm1 */
if (sa < 0)
{
mpn_add_n(v2, v2, vm1, kk1);
} else
{
mpn_sub_n(v2, v2, vm1, kk1);
}
ASSERT_NOCARRY (mpn_divexact_by3 (v2, v2, kk1)); /* v2 <- v2 / 3 */
/* vm1 <- t2 := (v1 - sa*vm1) / 2
*/
if (sa < 0)
{
#ifdef HAVE_NATIVE_mpn_rsh1add_n
mpn_rsh1add_n (vm1, v1, vm1, kk1);
#else
mpn_add_n (vm1, vm1, v1, kk1);
mpn_half (vm1, kk1);
#endif
}
else
{
#ifdef HAVE_NATIVE_mpn_rsh1sub_n
mpn_rsh1sub_n (vm1, v1, vm1, kk1);
#else
mpn_sub_n (vm1, v1, vm1, kk1);
mpn_half (vm1, kk1);
#endif
}
/* v1 <- v1 - v0 - vinf */
saved = c4[0];
c4[0] = vinf0;
#if HAVE_NATIVE_mpn_subadd_n
cy = mpn_subadd_n(v1, v1, v0, c4, rr2);
#else
cy = mpn_sub_n(v1, v1, v0, rr2);
cy += mpn_sub_n(v1, v1, c4, rr2);
#endif
c4[0] = saved;
if (rr2 < twok)
{
v1[twok] -= mpn_sub_n(v1 + rr2, v1 + rr2, v0 + rr2, twok - rr2);
MPN_DECR_U(v1 + rr2, kk1 - rr2, cy);
}
else v1[twok] -= cy;
saved = c4[0];
c4[0] = vinf0;
/* subtract 5*vinf from v2,
*/
cy = mpn_submul_1 (v2, c4, rr2, CNST_LIMB(5));
MPN_DECR_U (v2 + rr2, kk1 - rr2, cy);
c4[0] = saved;
/* v2 = (v2 - v1)/2 (exact)
*/
#ifdef HAVE_NATIVE_mpn_rsh1sub_n
mpn_rsh1sub_n (v2, v2, v1, kk1);
#else
mpn_sub_n (v2, v2, v1, kk1);
mpn_half (v2, kk1);
#endif
/* v1 = v1 - vm1
*/
mpn_sub_n(v1, v1, vm1, kk1);
/* vm1 = vm1 - v2 and add vm1 in {c+k, ...} */
#if HAVE_NATIVE_mpn_addsub_n
cy = mpn_addsub_n(c1, c1, vm1, v2, kk1);
#else
mpn_sub_n(vm1, vm1, v2, kk1);
cy = mpn_add_n (c1, c1, vm1, kk1);
#endif
ASSERT_NOCARRY (mpn_add_1(c3 + 1, c3 + 1, rr2 + k - 1, cy)); /* 4k+rr2-(3k+1) = rr2+k-1 */
/* don't forget to add vinf0 in {c+4k, ...} */
ASSERT_NOCARRY (mpn_add_1(c4, c4, rr2, vinf0));
/* add v2 in {c+3k, ...} */
if (rr2 <= k + 1)
ASSERT_NOCARRY (mpn_add_n (c3, c3, v2, k+rr2));
else
{
cy = mpn_add_n (c3, c3, v2, kk1);
if (cy) ASSERT_NOCARRY (mpn_add_1(c5 + 1, c5 + 1, rr2 - k - 1, cy)); /* 4k+rr2-(5k+1) = rr2-k-1 */
}
#undef v0
}
/*
We have a 3x2 blocked multiplication and therefore the output is of length
4 blocks. Therefore we evaluate at the 4 points 0, inf, -1, 1, i.e. we need
(a0*b0), (a2*b1), (a0-a1+a2)*(b0-b1), (a0+a1+a2)*(b0+b1).
The multiplication will be (2k+r) x (k + r2) and therefore the output has
space for 3k + rr2 limbs.
*/
void
mpn_toom32_mul (mp_ptr c, mp_srcptr a, mp_size_t an, mp_srcptr b, mp_size_t bn, mp_ptr t)
{
mp_size_t k, k1, kk1, r, r2, twok, threek, rr2, n1, n2;
mp_limb_t cy, cc, saved;
mp_ptr trec;
int sa, sb;
mp_ptr c1, c2, c3, c4, c5, t1, t2, t3, t4;
ASSERT(GMP_NUMB_BITS >= 6);
k = (an + 2) / 3; /* ceil(an/3) */
ASSERT(bn > k);
ASSERT(an >= 20);
twok = 2 * k;
threek = 3 * k;
k1 = k + 1;
kk1 = k + k1;
r = an - twok; /* last chunk */
r2 = bn - k; /* last chunk */
rr2 = r + r2;
c1 = c + k;
c2 = c1 + k;
c3 = c2 + k;
c4 = c3 + k;
c5 = c4 + k;
t1 = t + k;
t2 = t1 + k;
t3 = t2 + k;
t4 = t3 + k;
trec = t + 3 * k + 3;
/* put a0+a2 in {t, k+1}, and b0+b1 in {t2 + 2, k+1};
put a0+a1+a2 in {t1 + 1, k+1}
*/
cy = mpn_add_n (t, a, a + twok, r);
t3[2] = mpn_add_n (t2 + 2, b, b + k, r2);
if (r < k)
{
cy = mpn_add_1 (t + r, a + r, k - r, cy);
}
if (r2 < k)
{
t3[2] = mpn_add_1 (t2 + 2 + r2, b + r2, k - r2, t3[2]);
}
t2[1] = (t1[0] = cy) + mpn_add_n (t1 + 1, t, a + k, k);
/* compute v1 := (a0+a1+a2)*(b0+b1) in {c1, 2k+1};
since v1 < 6*B^(2k), v1 uses only 2k+1 words if GMP_NUMB_BITS >= 3 */
TOOM3_MUL_REC (c1, t1 + 1, t2 + 2, k1, trec);
saved = c1[0];
/* {c,2k} {c+2k,2k+1} {c+4k+1,r+r2-1}
v1
*/
/* put |a0-a1+a2| in {c0, k+1} and |b0-b1| in {t2 + 2,k+1} */
/* sa = sign(a0-a1+a2) */
/* sb = sign(b0-b1) */
sa = (t[k] != 0) ? 1 : mpn_cmp (t, a + k, k);
if (sa >= 0) c[k] = t[k] - mpn_sub_n (c, t, a + k, k);
else c[k] = -mpn_sub_n (c, a + k, t, k);
n1 = k;
n2 = r2;
MPN_NORMALIZE(b, n1);
MPN_NORMALIZE(b+k, n2);
if (n1 != n2) sb = (n1 > n2) ? 1 : -1;
else sb = mpn_cmp (b, b + k, n2);
if (sb >= 0)
{
t3[2] = mpn_sub_n (t2 + 2, b, b + k, r2);
if (k > r2) t3[2] = -mpn_sub_1(t2 + 2 + r2, b + r2, k - r2, t3[2]);
} else
{
mpn_sub_n (t2 + 2, b + k, b, r2);
MPN_ZERO(t2 + r2 + 2, k1 - r2);
}
sa *= sb; /* sign of vm1 */
/* compute vm1 := (a0-a1+a2)*(b0-b1) in {t, 2k+1};
since |vm1| < 2*B^(2k), vm1 uses only 2k+1 limbs */
TOOM3_MUL_REC (t, t2 + 2, c, k1, trec);
/* {c,2k} {c+2k,2k+1} {c+4k+1,r+r2-1}
v1
{t, 2k+1} {t+2k+1, 2k + 1}
vm1
*/
c1[0] = saved;
/* {c,k} {c+k,2k+1} {c+3k+1,r+r2-1}
v1
{t, 2k+1} {t+2k+1, 2k + 2}
vm1
*/
/* Compute vm1 <-- (vm1 + v1)/2 (note vm1 + v1 is positive) */
if (sa > 0)
{
#if HAVE_NATIVE_mpn_rsh1add_n
mpn_rsh1add_n(t, t, c1, kk1);
#else
mpn_add_n(t, t, c1, kk1);
mpn_half(t, kk1);
#endif
} else
{
#if HAVE_NATIVE_mpn_rsh1sub_n
mpn_rsh1sub_n(t, c1, t, kk1);
#else
mpn_sub_n(t, c1, t, kk1);
mpn_half(t, kk1);
#endif
}
/* Compute v1 <-- v1 - vm1 */
mpn_sub_n(c1, c1, t, kk1);
/* Note we could technically overflow
the end of the output if we add
everything in place without subtracting
the right things first. We get around
this by throwing away any high limbs
and carries, which must of necessity
cancel.
First we add vm1 in its place...
*/
n1 = kk1;
MPN_NORMALIZE(t, n1);
if (n1 >= k + rr2) /* if > here, high limb of vm1 and carry may be discarded */
{
cy = mpn_add_n(c2, c2, t, k1);
mpn_add_1(c3 + 1, t + k1, rr2 - 1, cy);
n2 = threek + rr2;
} else
{
c2[k1] = mpn_add_n(c2, c2, t, k1);
if (n1 > k1) c2[n1] = mpn_add_1(c3 + 1, t + k1, n1 - k1, c2[k1]);
n2 = twok + MAX(n1, k1) + 1;
}
/* Compute vinf := a2*b1 in {t, rr2} */
if (r == r2) TOOM3_MUL_REC (t, a + twok, b + k, r, trec);
else if (r > r2) mpn_mul(t, a + twok, r, b + k, r2);
else mpn_mul(t, b + k, r2, a + twok, r);
/* Add vinf into place */
cy = mpn_add_n(c3, c3, t, n2 - threek);
if (rr2 + threek > n2)
mpn_add_1(c + n2, t + n2 - threek, rr2 + threek - n2, cy);
/* v1 <-- v1 - vinf */
cy = mpn_sub_n(c1, c1, t, rr2);
if (cy) mpn_sub_1(c1 + rr2, c1 + rr2, twok, cy);
/* compute v0 := a0*b0 in {t, 2k} */
TOOM3_MUL_REC (t, a, b, k, trec);
/* Add v0 into place */
MPN_COPY(c, t, k);
cy = mpn_add_n(c + k, c + k, t + k, k);
if (cy) mpn_add_1(c + twok, c + twok, k + rr2, cy);
/* vm1 <-- vm1 - v0 */
if (twok >= k + rr2)
mpn_sub_n(c2, c2, t, k + rr2);
else
{
cy = mpn_sub_n(c2, c2, t, twok);
mpn_sub_1(c4, c4, rr2 + k - twok, cy);
}
}
