static int
tfm_dh_compute_key(unsigned char *shared, const BIGNUM * pub, DH *dh)
{
fp_int s, priv_key, p, peer_pub;
size_t size = 0;
int ret;
if (dh->pub_key == NULL || dh->g == NULL || dh->priv_key == NULL)
return -1;
fp_init(&p);
BN2mpz(&p, dh->p);
fp_init(&peer_pub);
BN2mpz(&peer_pub, pub);
/* check if peers pubkey is reasonable */
if (fp_isneg(&peer_pub)
|| fp_cmp(&peer_pub, &p) >= 0
|| fp_cmp_d(&peer_pub, 1) <= 0)
{
fp_zero(&p);
fp_zero(&peer_pub);
return -1;
}
fp_init(&priv_key);
BN2mpz(&priv_key, dh->priv_key);
fp_init(&s);
ret = fp_exptmod(&peer_pub, &priv_key, &p, &s);
fp_zero(&p);
fp_zero(&peer_pub);
fp_zero(&priv_key);
if (ret != 0)
return -1;
size = fp_unsigned_bin_size(&s);
fp_to_unsigned_bin(&s, shared);
fp_zero(&s);
return size;
}
static int
ltm_dh_compute_key(unsigned char *shared, const BIGNUM * pub, DH *dh)
{
mp_int s, priv_key, p, peer_pub;
int ret;
if (dh->pub_key == NULL || dh->g == NULL || dh->priv_key == NULL)
return -1;
mp_init_multi(&s, &priv_key, &p, &peer_pub, NULL);
BN2mpz(&p, dh->p);
BN2mpz(&peer_pub, pub);
/* check if peers pubkey is reasonable */
if (mp_isneg(&peer_pub)
|| mp_cmp(&peer_pub, &p) >= 0
|| mp_cmp_d(&peer_pub, 1) <= 0)
{
ret = -1;
goto out;
}
BN2mpz(&priv_key, dh->priv_key);
ret = mp_exptmod(&peer_pub, &priv_key, &p, &s);
if (ret != 0) {
ret = -1;
goto out;
}
ret = mp_unsigned_bin_size(&s);
mp_to_unsigned_bin(&s, shared);
out:
mp_clear_multi(&s, &priv_key, &p, &peer_pub, NULL);
return ret;
}
static int
ltm_dh_generate_key(DH *dh)
{
mp_int pub, priv_key, g, p;
int have_private_key = (dh->priv_key != NULL);
int codes, times = 0;
int res;
if (dh->p == NULL || dh->g == NULL)
return 0;
while (times++ < DH_NUM_TRIES) {
if (!have_private_key) {
size_t bits = BN_num_bits(dh->p);
if (dh->priv_key)
BN_free(dh->priv_key);
dh->priv_key = BN_new();
if (dh->priv_key == NULL)
return 0;
if (!BN_rand(dh->priv_key, bits - 1, 0, 0)) {
BN_clear_free(dh->priv_key);
dh->priv_key = NULL;
return 0;
}
}
if (dh->pub_key)
BN_free(dh->pub_key);
mp_init_multi(&pub, &priv_key, &g, &p, NULL);
BN2mpz(&priv_key, dh->priv_key);
BN2mpz(&g, dh->g);
BN2mpz(&p, dh->p);
res = mp_exptmod(&g, &priv_key, &p, &pub);
mp_clear_multi(&priv_key, &g, &p, NULL);
if (res != 0)
continue;
dh->pub_key = mpz2BN(&pub);
mp_clear(&pub);
if (dh->pub_key == NULL)
return 0;
if (DH_check_pubkey(dh, dh->pub_key, &codes) && codes == 0)
break;
if (have_private_key)
return 0;
}
if (times >= DH_NUM_TRIES) {
if (!have_private_key && dh->priv_key) {
BN_free(dh->priv_key);
dh->priv_key = NULL;
}
if (dh->pub_key) {
BN_free(dh->pub_key);
dh->pub_key = NULL;
}
return 0;
}
return 1;
}
static int
ltm_rsa_generate_key(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb)
{
mp_int el, p, q, n, d, dmp1, dmq1, iqmp, t1, t2, t3;
int counter, ret, bitsp;
if (bits < 789)
return -1;
bitsp = (bits + 1) / 2;
ret = -1;
mp_init_multi(&el, &p, &q, &n, &d,
&dmp1, &dmq1, &iqmp,
&t1, &t2, &t3, NULL);
BN2mpz(&el, e);
/* generate p and q so that p != q and bits(pq) ~ bits */
counter = 0;
do {
BN_GENCB_call(cb, 2, counter++);
CHECK(random_num(&p, bitsp), 0);
CHECK(mp_find_prime(&p), MP_YES);
mp_sub_d(&p, 1, &t1);
mp_gcd(&t1, &el, &t2);
} while(mp_cmp_d(&t2, 1) != 0);
BN_GENCB_call(cb, 3, 0);
counter = 0;
do {
BN_GENCB_call(cb, 2, counter++);
CHECK(random_num(&q, bits - bitsp), 0);
CHECK(mp_find_prime(&q), MP_YES);
if (mp_cmp(&p, &q) == 0) /* don't let p and q be the same */
continue;
mp_sub_d(&q, 1, &t1);
mp_gcd(&t1, &el, &t2);
} while(mp_cmp_d(&t2, 1) != 0);
/* make p > q */
if (mp_cmp(&p, &q) < 0) {
mp_int c;
c = p;
p = q;
q = c;
}
BN_GENCB_call(cb, 3, 1);
/* calculate n, n = p * q */
mp_mul(&p, &q, &n);
/* calculate d, d = 1/e mod (p - 1)(q - 1) */
mp_sub_d(&p, 1, &t1);
mp_sub_d(&q, 1, &t2);
mp_mul(&t1, &t2, &t3);
mp_invmod(&el, &t3, &d);
/* calculate dmp1 dmp1 = d mod (p-1) */
mp_mod(&d, &t1, &dmp1);
/* calculate dmq1 dmq1 = d mod (q-1) */
mp_mod(&d, &t2, &dmq1);
/* calculate iqmp iqmp = 1/q mod p */
mp_invmod(&q, &p, &iqmp);
/* fill in RSA key */
rsa->e = mpz2BN(&el);
rsa->p = mpz2BN(&p);
rsa->q = mpz2BN(&q);
rsa->n = mpz2BN(&n);
rsa->d = mpz2BN(&d);
rsa->dmp1 = mpz2BN(&dmp1);
rsa->dmq1 = mpz2BN(&dmq1);
rsa->iqmp = mpz2BN(&iqmp);
ret = 1;
out:
mp_clear_multi(&el, &p, &q, &n, &d,
&dmp1, &dmq1, &iqmp,
&t1, &t2, &t3, NULL);
return ret;
}
static int
ltm_rsa_private_decrypt(int flen, const unsigned char* from,
unsigned char* to, RSA* rsa, int padding)
{
unsigned char *ptr;
int res, size;
mp_int in, out, n, e, b, bi;
int blinding = (rsa->flags & RSA_FLAG_NO_BLINDING) == 0;
int do_unblind = 0;
if (padding != RSA_PKCS1_PADDING)
return -1;
size = RSA_size(rsa);
if (flen > size)
return -2;
mp_init_multi(&in, &n, &e, &out, &b, &bi, NULL);
BN2mpz(&n, rsa->n);
BN2mpz(&e, rsa->e);
mp_read_unsigned_bin(&in, rk_UNCONST(from), flen);
if(mp_isneg(&in) || mp_cmp(&in, &n) >= 0) {
size = -2;
goto out;
}
if (blinding) {
setup_blind(&n, &b, &bi);
blind(&in, &b, &e, &n);
do_unblind = 1;
}
if (rsa->p && rsa->q && rsa->dmp1 && rsa->dmq1 && rsa->iqmp) {
mp_int p, q, dmp1, dmq1, iqmp;
mp_init_multi(&p, &q, &dmp1, &dmq1, &iqmp, NULL);
BN2mpz(&p, rsa->p);
BN2mpz(&q, rsa->q);
BN2mpz(&dmp1, rsa->dmp1);
BN2mpz(&dmq1, rsa->dmq1);
BN2mpz(&iqmp, rsa->iqmp);
res = ltm_rsa_private_calculate(&in, &p, &q, &dmp1, &dmq1, &iqmp, &out);
mp_clear_multi(&p, &q, &dmp1, &dmq1, &iqmp, NULL);
if (res != 0) {
size = -3;
goto out;
}
} else {
mp_int d;
if(mp_isneg(&in) || mp_cmp(&in, &n) >= 0)
return -4;
BN2mpz(&d, rsa->d);
res = mp_exptmod(&in, &d, &n, &out);
mp_clear(&d);
if (res != 0) {
size = -5;
goto out;
}
}
if (do_unblind)
unblind(&out, &bi, &n);
ptr = to;
{
size_t ssize;
ssize = mp_unsigned_bin_size(&out);
assert(size >= ssize);
mp_to_unsigned_bin(&out, ptr);
size = ssize;
}
/* head zero was skipped by mp_int_to_unsigned */
if (*ptr != 2) {
size = -6;
goto out;
}
size--; ptr++;
while (size && *ptr != 0) {
size--; ptr++;
}
if (size == 0)
return -7;
size--; ptr++;
memmove(to, ptr, size);
out:
mp_clear_multi(&e, &n, &in, &out, &b, &bi, NULL);
return size;
}
static int
ltm_rsa_private_encrypt(int flen, const unsigned char* from,
unsigned char* to, RSA* rsa, int padding)
{
unsigned char *p, *p0;
int res;
int size;
mp_int in, out, n, e;
mp_int bi, b;
int blinding = (rsa->flags & RSA_FLAG_NO_BLINDING) == 0;
int do_unblind = 0;
if (padding != RSA_PKCS1_PADDING)
return -1;
mp_init_multi(&e, &n, &in, &out, &b, &bi, NULL);
size = RSA_size(rsa);
if (size < RSA_PKCS1_PADDING_SIZE || size - RSA_PKCS1_PADDING_SIZE < flen)
return -2;
p0 = p = malloc(size);
*p++ = 0;
*p++ = 1;
memset(p, 0xff, size - flen - 3);
p += size - flen - 3;
*p++ = 0;
memcpy(p, from, flen);
p += flen;
assert((p - p0) == size);
BN2mpz(&n, rsa->n);
BN2mpz(&e, rsa->e);
mp_read_unsigned_bin(&in, p0, size);
free(p0);
if(mp_isneg(&in) || mp_cmp(&in, &n) >= 0) {
size = -3;
goto out;
}
if (blinding) {
setup_blind(&n, &b, &bi);
blind(&in, &b, &e, &n);
do_unblind = 1;
}
if (rsa->p && rsa->q && rsa->dmp1 && rsa->dmq1 && rsa->iqmp) {
mp_int p, q, dmp1, dmq1, iqmp;
mp_init_multi(&p, &q, &dmp1, &dmq1, &iqmp, NULL);
BN2mpz(&p, rsa->p);
BN2mpz(&q, rsa->q);
BN2mpz(&dmp1, rsa->dmp1);
BN2mpz(&dmq1, rsa->dmq1);
BN2mpz(&iqmp, rsa->iqmp);
res = ltm_rsa_private_calculate(&in, &p, &q, &dmp1, &dmq1, &iqmp, &out);
mp_clear_multi(&p, &q, &dmp1, &dmq1, &iqmp, NULL);
if (res != 0) {
size = -4;
goto out;
}
} else {
mp_int d;
BN2mpz(&d, rsa->d);
res = mp_exptmod(&in, &d, &n, &out);
mp_clear(&d);
if (res != 0) {
size = -5;
goto out;
}
}
if (do_unblind)
unblind(&out, &bi, &n);
if (size > 0) {
size_t ssize;
ssize = mp_unsigned_bin_size(&out);
assert(size >= ssize);
mp_to_unsigned_bin(&out, to);
size = ssize;
}
out:
mp_clear_multi(&e, &n, &in, &out, &b, &bi, NULL);
return size;
}
static int
ltm_rsa_public_decrypt(int flen, const unsigned char* from,
unsigned char* to, RSA* rsa, int padding)
{
unsigned char *p;
int res;
size_t size;
mp_int s, us, n, e;
if (padding != RSA_PKCS1_PADDING)
return -1;
if (flen > RSA_size(rsa))
return -2;
mp_init_multi(&e, &n, &s, &us, NULL);
BN2mpz(&n, rsa->n);
BN2mpz(&e, rsa->e);
#if 0
/* Check that the exponent is larger then 3 */
if (mp_int_compare_value(&e, 3) <= 0) {
mp_clear_multi(&e, &n, &s, &us, NULL);
return -3;
}
#endif
mp_read_unsigned_bin(&s, rk_UNCONST(from), flen);
if (mp_cmp(&s, &n) >= 0) {
mp_clear_multi(&e, &n, &s, &us, NULL);
return -4;
}
res = mp_exptmod(&s, &e, &n, &us);
mp_clear_multi(&e, &n, &s, NULL);
if (res != 0) {
mp_clear(&us);
return -5;
}
p = to;
size = mp_unsigned_bin_size(&us);
assert(size <= RSA_size(rsa));
mp_to_unsigned_bin(&us, p);
mp_clear(&us);
/* head zero was skipped by mp_to_unsigned_bin */
if (*p == 0)
return -6;
if (*p != 1)
return -7;
size--; p++;
while (size && *p == 0xff) {
size--; p++;
}
if (size == 0 || *p != 0)
return -8;
size--; p++;
memmove(to, p, size);
return size;
}
static int
ltm_rsa_public_encrypt(int flen, const unsigned char* from,
unsigned char* to, RSA* rsa, int padding)
{
unsigned char *p, *p0;
int res;
size_t size, padlen;
mp_int enc, dec, n, e;
if (padding != RSA_PKCS1_PADDING)
return -1;
mp_init_multi(&n, &e, &enc, &dec, NULL);
size = RSA_size(rsa);
if (size < RSA_PKCS1_PADDING_SIZE || size - RSA_PKCS1_PADDING_SIZE < flen) {
mp_clear_multi(&n, &e, &enc, &dec);
return -2;
}
BN2mpz(&n, rsa->n);
BN2mpz(&e, rsa->e);
p = p0 = malloc(size - 1);
if (p0 == NULL) {
mp_clear_multi(&e, &n, &enc, &dec, NULL);
return -3;
}
padlen = size - flen - 3;
*p++ = 2;
if (RAND_bytes(p, padlen) != 1) {
mp_clear_multi(&e, &n, &enc, &dec, NULL);
free(p0);
return -4;
}
while(padlen) {
if (*p == 0)
*p = 1;
padlen--;
p++;
}
*p++ = 0;
memcpy(p, from, flen);
p += flen;
assert((p - p0) == size - 1);
mp_read_unsigned_bin(&dec, p0, size - 1);
free(p0);
res = mp_exptmod(&dec, &e, &n, &enc);
mp_clear_multi(&dec, &e, &n, NULL);
if (res != 0) {
mp_clear(&enc);
return -4;
}
{
size_t ssize;
ssize = mp_unsigned_bin_size(&enc);
assert(size >= ssize);
mp_to_unsigned_bin(&enc, to);
size = ssize;
}
mp_clear(&enc);
return size;
}
static int
tfm_rsa_private_encrypt(int flen, const unsigned char* from,
unsigned char* to, RSA* rsa, int padding)
{
unsigned char *p, *p0;
int res;
int size;
fp_int in, out, n, e;
if (padding != RSA_PKCS1_PADDING)
return -1;
size = RSA_size(rsa);
if (size < RSA_PKCS1_PADDING_SIZE || size - RSA_PKCS1_PADDING_SIZE < flen)
return -2;
p0 = p = malloc(size);
*p++ = 0;
*p++ = 1;
memset(p, 0xff, size - flen - 3);
p += size - flen - 3;
*p++ = 0;
memcpy(p, from, flen);
p += flen;
assert((p - p0) == size);
BN2mpz(&n, rsa->n);
BN2mpz(&e, rsa->e);
fp_init_multi(&in, &out, NULL);
fp_read_unsigned_bin(&in, p0, size);
free(p0);
if(fp_isneg(&in) || fp_cmp(&in, &n) >= 0) {
size = -3;
goto out;
}
if (rsa->p && rsa->q && rsa->dmp1 && rsa->dmq1 && rsa->iqmp) {
fp_int p, q, dmp1, dmq1, iqmp;
BN2mpz(&p, rsa->p);
BN2mpz(&q, rsa->q);
BN2mpz(&dmp1, rsa->dmp1);
BN2mpz(&dmq1, rsa->dmq1);
BN2mpz(&iqmp, rsa->iqmp);
res = tfm_rsa_private_calculate(&in, &p, &q, &dmp1, &dmq1, &iqmp, &out);
fp_zero_multi(&p, &q, &dmp1, &dmq1, &iqmp, NULL);
if (res != 0) {
size = -4;
goto out;
}
} else {
fp_int d;
BN2mpz(&d, rsa->d);
res = fp_exptmod(&in, &d, &n, &out);
fp_zero(&d);
if (res != 0) {
size = -5;
goto out;
}
}
if (size > 0) {
size_t ssize;
ssize = fp_unsigned_bin_size(&out);
assert(size >= ssize);
fp_to_unsigned_bin(&out, to);
size = ssize;
}
out:
fp_zero_multi(&e, &n, &in, &out, NULL);
return size;
}
static int
tfm_rsa_private_decrypt(int flen, const unsigned char* from,
unsigned char* to, RSA* rsa, int padding)
{
unsigned char *ptr;
int res;
int size;
fp_int in, out, n, e;
if (padding != RSA_PKCS1_PADDING)
return -1;
size = RSA_size(rsa);
if (flen > size)
return -2;
fp_init_multi(&in, &out, NULL);
BN2mpz(&n, rsa->n);
BN2mpz(&e, rsa->e);
fp_read_unsigned_bin(&in, rk_UNCONST(from), flen);
if(fp_isneg(&in) || fp_cmp(&in, &n) >= 0) {
size = -2;
goto out;
}
if (rsa->p && rsa->q && rsa->dmp1 && rsa->dmq1 && rsa->iqmp) {
fp_int p, q, dmp1, dmq1, iqmp;
BN2mpz(&p, rsa->p);
BN2mpz(&q, rsa->q);
BN2mpz(&dmp1, rsa->dmp1);
BN2mpz(&dmq1, rsa->dmq1);
BN2mpz(&iqmp, rsa->iqmp);
res = tfm_rsa_private_calculate(&in, &p, &q, &dmp1, &dmq1, &iqmp, &out);
fp_zero_multi(&p, &q, &dmp1, &dmq1, &iqmp, NULL);
if (res != 0) {
size = -3;
goto out;
}
} else {
fp_int d;
if(fp_isneg(&in) || fp_cmp(&in, &n) >= 0)
return -4;
BN2mpz(&d, rsa->d);
res = fp_exptmod(&in, &d, &n, &out);
fp_zero(&d);
if (res != 0) {
size = -5;
goto out;
}
}
ptr = to;
{
size_t ssize;
ssize = fp_unsigned_bin_size(&out);
assert(size >= ssize);
fp_to_unsigned_bin(&out, ptr);
size = ssize;
}
/* head zero was skipped by mp_int_to_unsigned */
if (*ptr != 2) {
size = -6;
goto out;
}
size--; ptr++;
while (size && *ptr != 0) {
size--; ptr++;
}
if (size == 0)
return -7;
size--; ptr++;
memmove(to, ptr, size);
out:
fp_zero_multi(&e, &n, &in, &out, NULL);
return size;
}
static int
gmp_rsa_generate_key(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb)
{
mpz_t el, p, q, n, d, dmp1, dmq1, iqmp, t1, t2, t3;
int counter, ret;
if (bits < 789)
return -1;
ret = -1;
mpz_init(el);
mpz_init(p);
mpz_init(q);
mpz_init(n);
mpz_init(d);
mpz_init(dmp1);
mpz_init(dmq1);
mpz_init(iqmp);
mpz_init(t1);
mpz_init(t2);
mpz_init(t3);
BN2mpz(el, e);
/* generate p and q so that p != q and bits(pq) ~ bits */
counter = 0;
do {
BN_GENCB_call(cb, 2, counter++);
random_num(p, bits / 2 + 1);
mpz_nextprime(p, p);
mpz_sub_ui(t1, p, 1);
mpz_gcd(t2, t1, el);
} while(mpz_cmp_ui(t2, 1) != 0);
BN_GENCB_call(cb, 3, 0);
counter = 0;
do {
BN_GENCB_call(cb, 2, counter++);
random_num(q, bits / 2 + 1);
mpz_nextprime(q, q);
mpz_sub_ui(t1, q, 1);
mpz_gcd(t2, t1, el);
} while(mpz_cmp_ui(t2, 1) != 0);
/* make p > q */
if (mpz_cmp(p, q) < 0)
mpz_swap(p, q);
BN_GENCB_call(cb, 3, 1);
/* calculate n, n = p * q */
mpz_mul(n, p, q);
/* calculate d, d = 1/e mod (p - 1)(q - 1) */
mpz_sub_ui(t1, p, 1);
mpz_sub_ui(t2, q, 1);
mpz_mul(t3, t1, t2);
mpz_invert(d, el, t3);
/* calculate dmp1 dmp1 = d mod (p-1) */
mpz_mod(dmp1, d, t1);
/* calculate dmq1 dmq1 = d mod (q-1) */
mpz_mod(dmq1, d, t2);
/* calculate iqmp iqmp = 1/q mod p */
mpz_invert(iqmp, q, p);
/* fill in RSA key */
rsa->e = mpz2BN(el);
rsa->p = mpz2BN(p);
rsa->q = mpz2BN(q);
rsa->n = mpz2BN(n);
rsa->d = mpz2BN(d);
rsa->dmp1 = mpz2BN(dmp1);
rsa->dmq1 = mpz2BN(dmq1);
rsa->iqmp = mpz2BN(iqmp);
ret = 1;
mpz_clear(el);
mpz_clear(p);
mpz_clear(q);
mpz_clear(n);
mpz_clear(d);
mpz_clear(dmp1);
mpz_clear(dmq1);
mpz_clear(iqmp);
mpz_clear(t1);
mpz_clear(t2);
mpz_clear(t3);
return ret;
}
static int
gmp_rsa_private_decrypt(int flen, const unsigned char* from,
unsigned char* to, RSA* rsa, int padding)
{
unsigned char *ptr;
size_t size;
mpz_t in, out, n, e;
if (padding != RSA_PKCS1_PADDING)
return -1;
size = RSA_size(rsa);
if (flen > size)
return -2;
mpz_init(in);
mpz_init(out);
BN2mpz(n, rsa->n);
BN2mpz(e, rsa->e);
mpz_import(in, flen, 1, 1, 1, 0, from);
if(mpz_cmp_ui(in, 0) < 0 ||
mpz_cmp(in, n) >= 0) {
size = 0;
goto out;
}
if (rsa->p && rsa->q && rsa->dmp1 && rsa->dmq1 && rsa->iqmp) {
mpz_t p, q, dmp1, dmq1, iqmp;
BN2mpz(p, rsa->p);
BN2mpz(q, rsa->q);
BN2mpz(dmp1, rsa->dmp1);
BN2mpz(dmq1, rsa->dmq1);
BN2mpz(iqmp, rsa->iqmp);
rsa_private_calculate(in, p, q, dmp1, dmq1, iqmp, out);
mpz_clear(p);
mpz_clear(q);
mpz_clear(dmp1);
mpz_clear(dmq1);
mpz_clear(iqmp);
} else {
mpz_t d;
#if 0
if(mp_int_compare_zero(&in) < 0 ||
mp_int_compare(&in, &n) >= 0)
return MP_RANGE;
#endif
BN2mpz(d, rsa->d);
mpz_powm(out, in, d, n);
mpz_clear(d);
}
ptr = to;
{
size_t ssize;
mpz_export(ptr, &ssize, 1, 1, 1, 0, out);
assert(size >= ssize);
size = ssize;
}
/* head zero was skipped by mp_int_to_unsigned */
if (*ptr != 2)
return -3;
size--; ptr++;
while (size && *ptr != 0) {
size--; ptr++;
}
if (size == 0)
return -4;
size--; ptr++;
memmove(to, ptr, size);
out:
mpz_clear(e);
mpz_clear(n);
mpz_clear(in);
mpz_clear(out);
return size;
}
static int
gmp_rsa_private_encrypt(int flen, const unsigned char* from,
unsigned char* to, RSA* rsa, int padding)
{
unsigned char *p, *p0;
size_t size;
mpz_t in, out, n, e;
if (padding != RSA_PKCS1_PADDING)
return -1;
size = RSA_size(rsa);
if (size < RSA_PKCS1_PADDING_SIZE || size - RSA_PKCS1_PADDING_SIZE < flen)
return -2;
p0 = p = malloc(size);
*p++ = 0;
*p++ = 1;
memset(p, 0xff, size - flen - 3);
p += size - flen - 3;
*p++ = 0;
memcpy(p, from, flen);
p += flen;
assert((p - p0) == size);
BN2mpz(n, rsa->n);
BN2mpz(e, rsa->e);
mpz_init(in);
mpz_init(out);
mpz_import(in, size, 1, 1, 1, 0, p0);
free(p0);
#if 0
if(mp_int_compare_zero(&in) < 0 ||
mp_int_compare(&in, &n) >= 0) {
size = 0;
goto out;
}
#endif
if (rsa->p && rsa->q && rsa->dmp1 && rsa->dmq1 && rsa->iqmp) {
mpz_t p, q, dmp1, dmq1, iqmp;
BN2mpz(p, rsa->p);
BN2mpz(q, rsa->q);
BN2mpz(dmp1, rsa->dmp1);
BN2mpz(dmq1, rsa->dmq1);
BN2mpz(iqmp, rsa->iqmp);
rsa_private_calculate(in, p, q, dmp1, dmq1, iqmp, out);
mpz_clear(p);
mpz_clear(q);
mpz_clear(dmp1);
mpz_clear(dmq1);
mpz_clear(iqmp);
} else {
mpz_t d;
BN2mpz(d, rsa->d);
mpz_powm(out, in, d, n);
mpz_clear(d);
}
{
size_t ssize;
mpz_export(to, &ssize, 1, 1, 1, 0, out);
assert(size >= ssize);
size = ssize;
}
mpz_clear(e);
mpz_clear(n);
mpz_clear(in);
mpz_clear(out);
return size;
}
static int
gmp_rsa_public_decrypt(int flen, const unsigned char* from,
unsigned char* to, RSA* rsa, int padding)
{
unsigned char *p;
size_t size;
mpz_t s, us, n, e;
if (padding != RSA_PKCS1_PADDING)
return -1;
if (flen > RSA_size(rsa))
return -2;
BN2mpz(n, rsa->n);
BN2mpz(e, rsa->e);
#if 0
/* Check that the exponent is larger then 3 */
if (mp_int_compare_value(&e, 3) <= 0) {
mp_int_clear(&n);
mp_int_clear(&e);
return -3;
}
#endif
mpz_init(s);
mpz_init(us);
mpz_import(s, flen, 1, 1, 1, 0, rk_UNCONST(from));
if (mpz_cmp(s, n) >= 0) {
mpz_clear(n);
mpz_clear(e);
return -4;
}
mpz_powm(us, s, e, n);
mpz_clear(s);
mpz_clear(n);
mpz_clear(e);
p = to;
mpz_export(p, &size, 1, 1, 1, 0, us);
assert(size <= RSA_size(rsa));
mpz_clear(us);
/* head zero was skipped by mp_int_to_unsigned */
if (*p == 0)
return -6;
if (*p != 1)
return -7;
size--; p++;
while (size && *p == 0xff) {
size--; p++;
}
if (size == 0 || *p != 0)
return -8;
size--; p++;
memmove(to, p, size);
return size;
}
static int
gmp_rsa_public_encrypt(int flen, const unsigned char* from,
unsigned char* to, RSA* rsa, int padding)
{
unsigned char *p, *p0;
size_t size, padlen;
mpz_t enc, dec, n, e;
if (padding != RSA_PKCS1_PADDING)
return -1;
size = RSA_size(rsa);
if (size < RSA_PKCS1_PADDING_SIZE || size - RSA_PKCS1_PADDING_SIZE < flen)
return -2;
BN2mpz(n, rsa->n);
BN2mpz(e, rsa->e);
p = p0 = malloc(size - 1);
if (p0 == NULL) {
mpz_clear(e);
mpz_clear(n);
return -3;
}
padlen = size - flen - 3;
assert(padlen >= 8);
*p++ = 2;
if (RAND_bytes(p, padlen) != 1) {
mpz_clear(e);
mpz_clear(n);
free(p0);
return -4;
}
while(padlen) {
if (*p == 0)
*p = 1;
padlen--;
p++;
}
*p++ = 0;
memcpy(p, from, flen);
p += flen;
assert((p - p0) == size - 1);
mpz_init(enc);
mpz_init(dec);
mpz_import(dec, size - 1, 1, 1, 1, 0, p0);
free(p0);
mpz_powm(enc, dec, e, n);
mpz_clear(dec);
mpz_clear(e);
mpz_clear(n);
{
size_t ssize;
mpz_export(to, &ssize, 1, 1, 1, 0, enc);
assert(size >= ssize);
size = ssize;
}
mpz_clear(enc);
return size;
}
