void
mpn_toom4_sqr (mp_ptr pp,
mp_srcptr ap, mp_size_t an,
mp_ptr scratch)
{
mp_size_t n, s;
mp_limb_t cy;
#define a0 ap
#define a1 (ap + n)
#define a2 (ap + 2*n)
#define a3 (ap + 3*n)
n = (an + 3) >> 2;
s = an - 3 * n;
ASSERT (0 < s && s <= n);
/* NOTE: The multiplications to v2, vm2, vh and vm1 overwrites the
* following limb, so these must be computed in order, and we need a
* one limb gap to tp. */
#define v0 pp /* 2n */
#define v1 (pp + 2 * n) /* 2n+1 */
#define vinf (pp + 6 * n) /* s+t */
#define v2 scratch /* 2n+1 */
#define vm2 (scratch + 2 * n + 1) /* 2n+1 */
#define vh (scratch + 4 * n + 2) /* 2n+1 */
#define vm1 (scratch + 6 * n + 3) /* 2n+1 */
#define tp (scratch + 8*n + 5)
/* No overlap with v1 */
#define apx pp /* n+1 */
#define amx (pp + 4*n + 2) /* n+1 */
/* Total scratch need: 8*n + 5 + scratch for recursive calls. This
gives roughly 32 n/3 + log term. */
/* Compute apx = a0 + 2 a1 + 4 a2 + 8 a3 and amx = a0 - 2 a1 + 4 a2 - 8 a3. */
mpn_toom_eval_dgr3_pm2 (apx, amx, ap, n, s, tp);
TOOM4_SQR_REC (v2, apx, n + 1, tp); /* v2, 2n+1 limbs */
TOOM4_SQR_REC (vm2, amx, n + 1, tp); /* vm2, 2n+1 limbs */
/* Compute apx = 8 a0 + 4 a1 + 2 a2 + a3 = (((2*a0 + a1) * 2 + a2) * 2 + a3 */
#if HAVE_NATIVE_mpn_addlsh1_n
cy = mpn_addlsh1_n (apx, a1, a0, n);
cy = 2*cy + mpn_addlsh1_n (apx, a2, apx, n);
if (s < n)
{
mp_limb_t cy2;
cy2 = mpn_addlsh1_n (apx, a3, apx, s);
apx[n] = 2*cy + mpn_lshift (apx + s, apx + s, n - s, 1);
MPN_INCR_U (apx + s, n+1-s, cy2);
}
else
apx[n] = 2*cy + mpn_addlsh1_n (apx, a3, apx, n);
#else
cy = mpn_lshift (apx, a0, n, 1);
cy += mpn_add_n (apx, apx, a1, n);
cy = 2*cy + mpn_lshift (apx, apx, n, 1);
cy += mpn_add_n (apx, apx, a2, n);
cy = 2*cy + mpn_lshift (apx, apx, n, 1);
apx[n] = cy + mpn_add (apx, apx, n, a3, s);
#endif
ASSERT (apx[n] < 15);
TOOM4_SQR_REC (vh, apx, n + 1, tp); /* vh, 2n+1 limbs */
/* Compute apx = a0 + a1 + a2 + a3 and amx = a0 - a1 + a2 - a3. */
mpn_toom_eval_dgr3_pm1 (apx, amx, ap, n, s, tp);
TOOM4_SQR_REC (v1, apx, n + 1, tp); /* v1, 2n+1 limbs */
TOOM4_SQR_REC (vm1, amx, n + 1, tp); /* vm1, 2n+1 limbs */
TOOM4_SQR_REC (v0, a0, n, tp);
TOOM4_SQR_REC (vinf, a3, s, tp); /* vinf, 2s limbs */
mpn_toom_interpolate_7pts (pp, n, 0, vm2, vm1, v2, vh, 2*s, tp);
}
void
mpn_toom6h_mul (mp_ptr pp,
mp_srcptr ap, mp_size_t an,
mp_srcptr bp, mp_size_t bn, mp_ptr scratch)
{
mp_size_t n, s, t;
int p, q, half;
int sign;
/***************************** decomposition *******************************/
ASSERT (an >= bn);
/* Can not handle too much unbalancement */
ASSERT (bn >= 42);
/* Can not handle too much unbalancement */
ASSERT ((an*3 < bn * 8) || (bn >= 46 && an * 6 < bn * 17));
/* Limit num/den is a rational number between
(12/11)^(log(4)/log(2*4-1)) and (12/11)^(log(6)/log(2*6-1)) */
#define LIMIT_numerator (18)
#define LIMIT_denominat (17)
if (LIKELY (an * LIMIT_denominat < LIMIT_numerator * bn)) /* is 6*... < 6*... */
{
n = 1 + (an - 1) / (size_t) 6;
p = q = 5;
half = 0;
s = an - 5 * n;
t = bn - 5 * n;
}
else {
if (an * 5 * LIMIT_numerator < LIMIT_denominat * 7 * bn)
{ p = 7; q = 6; }
else if (an * 5 * LIMIT_denominat < LIMIT_numerator * 7 * bn)
{ p = 7; q = 5; }
else if (an * LIMIT_numerator < LIMIT_denominat * 2 * bn) /* is 4*... < 8*... */
{ p = 8; q = 5; }
else if (an * LIMIT_denominat < LIMIT_numerator * 2 * bn) /* is 4*... < 8*... */
{ p = 8; q = 4; }
else
{ p = 9; q = 4; }
half = (p ^ q) & 1;
n = 1 + (q * an >= p * bn ? (an - 1) / (size_t) p : (bn - 1) / (size_t) q);
p--; q--;
s = an - p * n;
t = bn - q * n;
/* With LIMIT = 16/15, the following recover is needed only if bn<=73*/
if (half) { /* Recover from badly chosen splitting */
if (UNLIKELY (s<1)) {p--; s+=n; half=0;}
else if (UNLIKELY (t<1)) {q--; t+=n; half=0;}
}
}
#undef LIMIT_numerator
#undef LIMIT_denominat
ASSERT (0 < s && s <= n);
ASSERT (0 < t && t <= n);
ASSERT (half || s + t > 3);
ASSERT (n > 2);
#define r4 (pp + 3 * n) /* 3n+1 */
#define r2 (pp + 7 * n) /* 3n+1 */
#define r0 (pp +11 * n) /* s+t <= 2*n */
#define r5 (scratch) /* 3n+1 */
#define r3 (scratch + 3 * n + 1) /* 3n+1 */
#define r1 (scratch + 6 * n + 2) /* 3n+1 */
#define v0 (pp + 7 * n) /* n+1 */
#define v1 (pp + 8 * n+1) /* n+1 */
#define v2 (pp + 9 * n+2) /* n+1 */
#define v3 (scratch + 9 * n + 3) /* n+1 */
#define wsi (scratch + 9 * n + 3) /* 3n+1 */
#define wse (scratch +10 * n + 4) /* 2n+1 */
/* Alloc also 3n+1 limbs for wsi... toom_interpolate_12pts may
need all of them */
/* if (scratch == NULL) */
/* scratch = TMP_SALLOC_LIMBS(mpn_toom6_sqr_itch(n * 6)); */
ASSERT (12 * n + 6 <= mpn_toom6h_mul_itch(an,bn));
ASSERT (12 * n + 6 <= mpn_toom6_sqr_itch(n * 6));
/********************** evaluation and recursive calls *********************/
/* $\pm1/2$ */
sign = mpn_toom_eval_pm2rexp (v2, v0, p, ap, n, s, 1, pp) ^
mpn_toom_eval_pm2rexp (v3, v1, q, bp, n, t, 1, pp);
/* A(-1/2)*B(-1/2)*2^. */ /* A(+1/2)*B(+1/2)*2^. */
TOOM6H_MUL_N_REC(pp, v0, v1, 2, r5, v2, v3, n + 1, wse);
mpn_toom_couple_handling (r5, 2 * n + 1, pp, sign, n, 1+half , half);
/* $\pm1$ */
sign = mpn_toom_eval_pm1 (v2, v0, p, ap, n, s, pp);
if (UNLIKELY (q == 3))
sign ^= mpn_toom_eval_dgr3_pm1 (v3, v1, bp, n, t, pp);
else
sign ^= mpn_toom_eval_pm1 (v3, v1, q, bp, n, t, pp);
/* A(-1)*B(-1) */ /* A(1)*B(1) */
TOOM6H_MUL_N_REC(pp, v0, v1, 2, r3, v2, v3, n + 1, wse);
mpn_toom_couple_handling (r3, 2 * n + 1, pp, sign, n, 0, 0);
/* $\pm4$ */
sign = mpn_toom_eval_pm2exp (v2, v0, p, ap, n, s, 2, pp) ^
mpn_toom_eval_pm2exp (v3, v1, q, bp, n, t, 2, pp);
/* A(-4)*B(-4) */
TOOM6H_MUL_N_REC(pp, v0, v1, 2, r1, v2, v3, n + 1, wse); /* A(+4)*B(+4) */
mpn_toom_couple_handling (r1, 2 * n + 1, pp, sign, n, 2, 4);
/* $\pm1/4$ */
sign = mpn_toom_eval_pm2rexp (v2, v0, p, ap, n, s, 2, pp) ^
mpn_toom_eval_pm2rexp (v3, v1, q, bp, n, t, 2, pp);
/* A(-1/4)*B(-1/4)*4^. */ /* A(+1/4)*B(+1/4)*4^. */
TOOM6H_MUL_N_REC(pp, v0, v1, 2, r4, v2, v3, n + 1, wse);
mpn_toom_couple_handling (r4, 2 * n + 1, pp, sign, n, 2*(1+half), 2*(half));
/* $\pm2$ */
sign = mpn_toom_eval_pm2 (v2, v0, p, ap, n, s, pp) ^
mpn_toom_eval_pm2 (v3, v1, q, bp, n, t, pp);
/* A(-2)*B(-2) */ /* A(+2)*B(+2) */
TOOM6H_MUL_N_REC(pp, v0, v1, 2, r2, v2, v3, n + 1, wse);
mpn_toom_couple_handling (r2, 2 * n + 1, pp, sign, n, 1, 2);
#undef v0
#undef v1
#undef v2
#undef v3
#undef wse
/* A(0)*B(0) */
TOOM6H_MUL_N_REC(pp, ap, bp, 0, pp, ap, bp, n, wsi);
/* Infinity */
if (UNLIKELY (half != 0)) {
if (s > t) {
TOOM6H_MUL_REC(r0, ap + p * n, s, bp + q * n, t, wsi);
} else {
TOOM6H_MUL_REC(r0, bp + q * n, t, ap + p * n, s, wsi);
};
};
mpn_toom_interpolate_12pts (pp, r1, r3, r5, n, s+t, half, wsi);
#undef r0
#undef r1
#undef r2
#undef r3
#undef r4
#undef r5
#undef wsi
}
void
mpn_toom44_mul (mp_ptr pp,
mp_srcptr ap, mp_size_t an,
mp_srcptr bp, mp_size_t bn,
mp_ptr scratch)
{
mp_size_t n, s, t;
mp_limb_t cy;
enum toom7_flags flags;
#define a0 ap
#define a1 (ap + n)
#define a2 (ap + 2*n)
#define a3 (ap + 3*n)
#define b0 bp
#define b1 (bp + n)
#define b2 (bp + 2*n)
#define b3 (bp + 3*n)
ASSERT (an >= bn);
n = (an + 3) >> 2;
s = an - 3 * n;
t = bn - 3 * n;
ASSERT (0 < s && s <= n);
ASSERT (0 < t && t <= n);
ASSERT (s >= t);
/* NOTE: The multiplications to v2, vm2, vh and vm1 overwrites the
* following limb, so these must be computed in order, and we need a
* one limb gap to tp. */
#define v0 pp /* 2n */
#define v1 (pp + 2 * n) /* 2n+1 */
#define vinf (pp + 6 * n) /* s+t */
#define v2 scratch /* 2n+1 */
#define vm2 (scratch + 2 * n + 1) /* 2n+1 */
#define vh (scratch + 4 * n + 2) /* 2n+1 */
#define vm1 (scratch + 6 * n + 3) /* 2n+1 */
#define tp (scratch + 8*n + 5)
/* apx and bpx must not overlap with v1 */
#define apx pp /* n+1 */
#define amx (pp + n + 1) /* n+1 */
#define bmx (pp + 2*n + 2) /* n+1 */
#define bpx (pp + 4*n + 2) /* n+1 */
/* Total scratch need: 8*n + 5 + scratch for recursive calls. This
gives roughly 32 n/3 + log term. */
/* Compute apx = a0 + 2 a1 + 4 a2 + 8 a3 and amx = a0 - 2 a1 + 4 a2 - 8 a3. */
flags = (enum toom7_flags) (toom7_w1_neg & mpn_toom_eval_dgr3_pm2 (apx, amx, ap, n, s, tp));
/* Compute bpx = b0 + 2 b1 + 4 b2 + 8 b3 and bmx = b0 - 2 b1 + 4 b2 - 8 b3. */
flags = (enum toom7_flags) (flags ^ toom7_w1_neg & mpn_toom_eval_dgr3_pm2 (bpx, bmx, bp, n, t, tp));
TOOM44_MUL_N_REC (v2, apx, bpx, n + 1, tp); /* v2, 2n+1 limbs */
TOOM44_MUL_N_REC (vm2, amx, bmx, n + 1, tp); /* vm2, 2n+1 limbs */
/* Compute apx = 8 a0 + 4 a1 + 2 a2 + a3 = (((2*a0 + a1) * 2 + a2) * 2 + a3 */
#if HAVE_NATIVE_mpn_addlsh1_n
cy = mpn_addlsh1_n (apx, a1, a0, n);
cy = 2*cy + mpn_addlsh1_n (apx, a2, apx, n);
if (s < n)
{
mp_limb_t cy2;
cy2 = mpn_addlsh1_n (apx, a3, apx, s);
apx[n] = 2*cy + mpn_lshift (apx + s, apx + s, n - s, 1);
MPN_INCR_U (apx + s, n+1-s, cy2);
}
else
apx[n] = 2*cy + mpn_addlsh1_n (apx, a3, apx, n);
#else
cy = mpn_lshift (apx, a0, n, 1);
cy += mpn_add_n (apx, apx, a1, n);
cy = 2*cy + mpn_lshift (apx, apx, n, 1);
cy += mpn_add_n (apx, apx, a2, n);
cy = 2*cy + mpn_lshift (apx, apx, n, 1);
apx[n] = cy + mpn_add (apx, apx, n, a3, s);
#endif
/* Compute bpx = 8 b0 + 4 b1 + 2 b2 + b3 = (((2*b0 + b1) * 2 + b2) * 2 + b3 */
#if HAVE_NATIVE_mpn_addlsh1_n
cy = mpn_addlsh1_n (bpx, b1, b0, n);
cy = 2*cy + mpn_addlsh1_n (bpx, b2, bpx, n);
if (t < n)
{
mp_limb_t cy2;
cy2 = mpn_addlsh1_n (bpx, b3, bpx, t);
bpx[n] = 2*cy + mpn_lshift (bpx + t, bpx + t, n - t, 1);
MPN_INCR_U (bpx + t, n+1-t, cy2);
}
else
bpx[n] = 2*cy + mpn_addlsh1_n (bpx, b3, bpx, n);
#else
cy = mpn_lshift (bpx, b0, n, 1);
cy += mpn_add_n (bpx, bpx, b1, n);
cy = 2*cy + mpn_lshift (bpx, bpx, n, 1);
cy += mpn_add_n (bpx, bpx, b2, n);
cy = 2*cy + mpn_lshift (bpx, bpx, n, 1);
bpx[n] = cy + mpn_add (bpx, bpx, n, b3, t);
#endif
ASSERT (apx[n] < 15);
ASSERT (bpx[n] < 15);
TOOM44_MUL_N_REC (vh, apx, bpx, n + 1, tp); /* vh, 2n+1 limbs */
/* Compute apx = a0 + a1 + a2 + a3 and amx = a0 - a1 + a2 - a3. */
flags = (enum toom7_flags) (flags | toom7_w3_neg & mpn_toom_eval_dgr3_pm1 (apx, amx, ap, n, s, tp));
/* Compute bpx = b0 + b1 + b2 + b3 bnd bmx = b0 - b1 + b2 - b3. */
flags = (enum toom7_flags) (flags ^ toom7_w3_neg & mpn_toom_eval_dgr3_pm1 (bpx, bmx, bp, n, t, tp));
TOOM44_MUL_N_REC (vm1, amx, bmx, n + 1, tp); /* vm1, 2n+1 limbs */
/* Clobbers amx, bmx. */
TOOM44_MUL_N_REC (v1, apx, bpx, n + 1, tp); /* v1, 2n+1 limbs */
TOOM44_MUL_N_REC (v0, a0, b0, n, tp);
if (s > t)
mpn_mul (vinf, a3, s, b3, t);
else
TOOM44_MUL_N_REC (vinf, a3, b3, s, tp); /* vinf, s+t limbs */
mpn_toom_interpolate_7pts (pp, n, flags, vm2, vm1, v2, vh, s + t, tp);
}
