int
rpmDigestFinal(DIGEST_CTX ctx, void ** datap, size_t *lenp, int asAscii)
{
unsigned char * digest;
unsigned int digestlen;
if (ctx == NULL)
return -1;
digestlen = HASH_ResultLenContext(ctx->hashctx);
digest = xmalloc(digestlen);
DPRINTF((stderr, "*** Final(%p,%p,%p,%zd) hashctx %p digest %p\n", ctx, datap, lenp, asAscii, ctx->hashctx, digest));
/* FIX: check rc */
HASH_End(ctx->hashctx, digest, (unsigned int *) &digestlen, digestlen);
/* Return final digest. */
if (!asAscii) {
if (lenp) *lenp = digestlen;
if (datap) {
*datap = digest;
digest = NULL;
}
} else {
if (lenp) *lenp = (2*digestlen) + 1;
if (datap) {
const uint8_t * s = (const uint8_t *) digest;
*datap = pgpHexStr(s, digestlen);
}
}
if (digest) {
memset(digest, 0, digestlen); /* In case it's sensitive */
free(digest);
}
HASH_Destroy(ctx->hashctx);
memset(ctx, 0, sizeof(*ctx)); /* In case it's sensitive */
free(ctx);
return 0;
}
SECStatus
HASH_HashBuf(HASH_HashType type,
unsigned char *dest,
const unsigned char *src,
PRUint32 src_len)
{
HASHContext *cx;
unsigned int part;
if ((type < HASH_AlgNULL) || (type >= HASH_AlgTOTAL)) {
return (SECFailure);
}
cx = HASH_Create(type);
if (cx == NULL) {
return (SECFailure);
}
HASH_Begin(cx);
HASH_Update(cx, src, src_len);
HASH_End(cx, dest, &part, HASH_ResultLenContext(cx));
HASH_Destroy(cx);
return (SECSuccess);
}
static int sha512_crypt_r(const char *key,
const char *salt,
char *buffer, size_t buflen)
{
unsigned char temp_result[64] __attribute__((__aligned__(ALIGN64)));
unsigned char alt_result[64] __attribute__((__aligned__(ALIGN64)));
size_t rounds = ROUNDS_DEFAULT;
bool rounds_custom = false;
HASHContext *alt_ctx = NULL;
HASHContext *ctx = NULL;
size_t salt_len;
size_t key_len;
size_t cnt;
char *copied_salt = NULL;
char *copied_key = NULL;
char *p_bytes = NULL;
char *s_bytes = NULL;
int p1, p2, p3, pt, n;
unsigned int part;
char *cp, *tmp;
int ret;
/* Find beginning of salt string. The prefix should normally always be
* present. Just in case it is not. */
if (strncmp(salt, sha512_salt_prefix, SALT_PREF_SIZE) == 0) {
/* Skip salt prefix. */
salt += SALT_PREF_SIZE;
}
if (strncmp(salt, sha512_rounds_prefix, ROUNDS_SIZE) == 0) {
unsigned long int srounds;
const char *num;
char *endp;
num = salt + ROUNDS_SIZE;
srounds = strtoul(num, &endp, 10);
if (*endp == '$') {
salt = endp + 1;
if (srounds < ROUNDS_MIN) srounds = ROUNDS_MIN;
if (srounds > ROUNDS_MAX) srounds = ROUNDS_MAX;
rounds = srounds;
rounds_custom = true;
}
}
salt_len = MIN(strcspn(salt, "$"), SALT_LEN_MAX);
key_len = strlen(key);
if ((PTR_2_INT(key) % ALIGN64) != 0) {
tmp = (char *)alloca(key_len + ALIGN64);
key = copied_key = memcpy(tmp + ALIGN64 - PTR_2_INT(tmp) % ALIGN64, key, key_len);
}
if (PTR_2_INT(salt) % ALIGN64 != 0) {
tmp = (char *)alloca(salt_len + ALIGN64);
salt = copied_salt = memcpy(tmp + ALIGN64 - PTR_2_INT(tmp) % ALIGN64, salt, salt_len);
}
ret = nspr_nss_init();
if (ret != EOK) {
ret = EIO;
goto done;
}
ctx = HASH_Create(HASH_AlgSHA512);
if (!ctx) {
ret = EIO;
goto done;
}
alt_ctx = HASH_Create(HASH_AlgSHA512);
if (!alt_ctx) {
ret = EIO;
goto done;
}
/* Prepare for the real work. */
HASH_Begin(ctx);
/* Add the key string. */
HASH_Update(ctx, (const unsigned char *)key, key_len);
/* The last part is the salt string. This must be at most 16
* characters and it ends at the first `$' character (for
* compatibility with existing implementations). */
HASH_Update(ctx, (const unsigned char *)salt, salt_len);
/* Compute alternate SHA512 sum with input KEY, SALT, and KEY.
* The final result will be added to the first context. */
HASH_Begin(alt_ctx);
/* Add key. */
HASH_Update(alt_ctx, (const unsigned char *)key, key_len);
/* Add salt. */
HASH_Update(alt_ctx, (const unsigned char *)salt, salt_len);
/* Add key again. */
HASH_Update(alt_ctx, (const unsigned char *)key, key_len);
/* Now get result of this (64 bytes) and add it to the other context. */
HASH_End(alt_ctx, alt_result, &part, HASH_ResultLenContext(alt_ctx));
/* Add for any character in the key one byte of the alternate sum. */
for (cnt = key_len; cnt > 64; cnt -= 64) {
HASH_Update(ctx, alt_result, 64);
}
HASH_Update(ctx, alt_result, cnt);
/* Take the binary representation of the length of the key and for every
* 1 add the alternate sum, for every 0 the key. */
for (cnt = key_len; cnt > 0; cnt >>= 1) {
if ((cnt & 1) != 0) {
HASH_Update(ctx, alt_result, 64);
} else {
HASH_Update(ctx, (const unsigned char *)key, key_len);
}
}
/* Create intermediate result. */
HASH_End(ctx, alt_result, &part, HASH_ResultLenContext(ctx));
/* Start computation of P byte sequence. */
HASH_Begin(alt_ctx);
/* For every character in the password add the entire password. */
for (cnt = 0; cnt < key_len; cnt++) {
HASH_Update(alt_ctx, (const unsigned char *)key, key_len);
}
/* Finish the digest. */
HASH_End(alt_ctx, temp_result, &part, HASH_ResultLenContext(alt_ctx));
/* Create byte sequence P. */
cp = p_bytes = alloca(key_len);
for (cnt = key_len; cnt >= 64; cnt -= 64) {
cp = mempcpy(cp, temp_result, 64);
}
memcpy(cp, temp_result, cnt);
/* Start computation of S byte sequence. */
HASH_Begin(alt_ctx);
/* For every character in the password add the entire salt. */
for (cnt = 0; cnt < 16 + alt_result[0]; cnt++) {
HASH_Update(alt_ctx, (const unsigned char *)salt, salt_len);
}
/* Finish the digest. */
HASH_End(alt_ctx, temp_result, &part, HASH_ResultLenContext(alt_ctx));
/* Create byte sequence S. */
cp = s_bytes = alloca(salt_len);
for (cnt = salt_len; cnt >= 64; cnt -= 64) {
cp = mempcpy(cp, temp_result, 64);
}
memcpy(cp, temp_result, cnt);
/* Repeatedly run the collected hash value through SHA512 to burn CPU cycles. */
for (cnt = 0; cnt < rounds; cnt++) {
HASH_Begin(ctx);
/* Add key or last result. */
if ((cnt & 1) != 0) {
HASH_Update(ctx, (const unsigned char *)p_bytes, key_len);
} else {
HASH_Update(ctx, alt_result, 64);
}
/* Add salt for numbers not divisible by 3. */
if (cnt % 3 != 0) {
HASH_Update(ctx, (const unsigned char *)s_bytes, salt_len);
}
/* Add key for numbers not divisible by 7. */
if (cnt % 7 != 0) {
HASH_Update(ctx, (const unsigned char *)p_bytes, key_len);
}
/* Add key or last result. */
if ((cnt & 1) != 0) {
HASH_Update(ctx, alt_result, 64);
} else {
HASH_Update(ctx, (const unsigned char *)p_bytes, key_len);
}
/* Create intermediate result. */
HASH_End(ctx, alt_result, &part, HASH_ResultLenContext(ctx));
}
/* Now we can construct the result string.
* It consists of three parts. */
if (buflen <= SALT_PREF_SIZE) {
ret = ERANGE;
goto done;
}
cp = __stpncpy(buffer, sha512_salt_prefix, SALT_PREF_SIZE);
buflen -= SALT_PREF_SIZE;
if (rounds_custom) {
n = snprintf(cp, buflen, "%s%zu$",
sha512_rounds_prefix, rounds);
if (n < 0 || n >= buflen) {
ret = ERANGE;
goto done;
}
cp += n;
buflen -= n;
}
if (buflen <= salt_len + 1) {
ret = ERANGE;
goto done;
}
cp = __stpncpy(cp, salt, salt_len);
*cp++ = '$';
buflen -= salt_len + 1;
/* fuzzyfill the base 64 string */
p1 = 0;
p2 = 21;
p3 = 42;
for (n = 0; n < 21; n++) {
b64_from_24bit(&cp, &buflen, 4, alt_result[p1], alt_result[p2], alt_result[p3]);
if (buflen == 0) {
ret = ERANGE;
goto done;
}
pt = p1;
p1 = p2 + 1;
p2 = p3 + 1;
p3 = pt + 1;
}
/* 64th and last byte */
b64_from_24bit(&cp, &buflen, 2, 0, 0, alt_result[p3]);
if (buflen == 0) {
ret = ERANGE;
goto done;
}
*cp = '\0';
ret = EOK;
done:
/* Clear the buffer for the intermediate result so that people attaching
* to processes or reading core dumps cannot get any information. We do it
* in this way to clear correct_words[] inside the SHA512 implementation
* as well. */
if (ctx) HASH_Destroy(ctx);
if (alt_ctx) HASH_Destroy(alt_ctx);
if (p_bytes) memset(p_bytes, '\0', key_len);
if (s_bytes) memset(s_bytes, '\0', salt_len);
if (copied_key) memset(copied_key, '\0', key_len);
if (copied_salt) memset(copied_salt, '\0', salt_len);
memset(temp_result, '\0', sizeof(temp_result));
return ret;
}
