/** * xmlSecOpenSSLGenerateRandom: * @buffer: the destination buffer. * @size: the numer of bytes to generate. * * Generates @size random bytes and puts result in @buffer. * * Returns: 0 on success or a negative value otherwise. */ int xmlSecOpenSSLGenerateRandom(xmlSecBufferPtr buffer, xmlSecSize size) { int ret; xmlSecAssert2(buffer != NULL, -1); xmlSecAssert2(size > 0, -1); ret = xmlSecBufferSetSize(buffer, size); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, NULL, "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", size); return(-1); } /* get random data */ ret = RAND_bytes((xmlSecByte*)xmlSecBufferGetData(buffer), size); if(ret != 1) { xmlSecOpenSSLError(NULL, "RAND_bytes"); return(-1); } return(0); }
PyObject *xmlsec_BufferSetSize(PyObject *self, PyObject *args) { PyObject *buf_obj; xmlSecBufferPtr buf; xmlSecSize size; if (CheckArgs(args, "OI:bufferSetSize")) { if (!PyArg_ParseTuple(args, "Oi:bufferSetSize", &buf_obj, &size)) return NULL; } else return NULL; buf = xmlSecBufferPtr_get(buf_obj); return (wrap_int(xmlSecBufferSetSize(buf, size))); }
/** * xmlSecMSCryptoGenerateRandom: * @buffer: the destination buffer. * @size: the numer of bytes to generate. * * Generates @size random bytes and puts result in @buffer * (not implemented yet). * * Returns: 0 on success or a negative value otherwise. */ int xmlSecMSCryptoGenerateRandom(xmlSecBufferPtr buffer, size_t size) { HCRYPTPROV hProv = 0; int ret; xmlSecAssert2(buffer != NULL, -1); xmlSecAssert2(size > 0, -1); ret = xmlSecBufferSetSize(buffer, size); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, NULL, "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", size); return(-1); } hProv = xmlSecMSCryptoFindProvider(xmlSecMSCryptoProviderInfo_Random, NULL, CRYPT_VERIFYCONTEXT, FALSE); if (0 == hProv) { xmlSecError(XMLSEC_ERRORS_HERE, NULL, "xmlSecMSCryptoFindProvider", XMLSEC_ERRORS_R_XMLSEC_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } if (FALSE == CryptGenRandom(hProv, (DWORD)size, xmlSecBufferGetData(buffer))) { xmlSecError(XMLSEC_ERRORS_HERE, NULL, "CryptGenRandom", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); CryptReleaseContext(hProv,0); return(-1); } CryptReleaseContext(hProv, 0); return(0); }
/** * xmlSecNssGenerateRandom: * @buffer: the destination buffer. * @size: the numer of bytes to generate. * * Generates @size random bytes and puts result in @buffer. * * Returns: 0 on success or a negative value otherwise. */ int xmlSecNssGenerateRandom(xmlSecBufferPtr buffer, xmlSecSize size) { SECStatus rv; int ret; xmlSecAssert2(buffer != NULL, -1); xmlSecAssert2(size > 0, -1); ret = xmlSecBufferSetSize(buffer, size); if(ret < 0) { xmlSecInternalError2("xmlSecBufferSetSize", NULL, "size=%d", size); return(-1); } /* get random data */ rv = PK11_GenerateRandom((xmlSecByte*)xmlSecBufferGetData(buffer), size); if(rv != SECSuccess) { xmlSecNssError2("PK11_GenerateRandom", NULL, "size=%lu", (unsigned long)size); return(-1); } return(0); }
static int xmlSecMSCryptoRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) { xmlSecMSCryptoRsaPkcs1CtxPtr ctx; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize; xmlSecSize keySize; int ret; HCRYPTKEY hKey = 0; DWORD dwInLen; DWORD dwBufLen; DWORD dwOutLen; BYTE * outBuf; BYTE * inBuf; int i; xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecMSCryptoTransformRsaPkcs1Id), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecMSCryptoRsaPkcs1Size), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecMSCryptoRsaPkcs1GetCtx(transform); xmlSecAssert2(ctx != NULL, -1); xmlSecAssert2(ctx->data != NULL, -1); keySize = xmlSecKeyDataGetSize(ctx->data) / 8; xmlSecAssert2(keySize > 0, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); /* the encoded size is equal to the keys size so we could not * process more than that */ if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "%d when expected less than %d", inSize, keySize); return(-1); } else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "%d when expected %d", inSize, keySize); return(-1); } outSize = keySize; ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { BYTE ch; if(inSize > outSize) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "inSize=%d;outSize=%d", inSize, outSize); return(-1); } ret = xmlSecBufferSetData(out, xmlSecBufferGetData(in), inSize); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetData", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", inSize); return(-1); } dwInLen = inSize; dwBufLen = outSize; if (0 == (hKey = xmlSecMSCryptoKeyDataGetKey(ctx->data, xmlSecKeyDataTypePublic))) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, NULL, "xmlSecMSCryptoKeyDataGetKey", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return (-1); } outBuf = xmlSecBufferGetData(out); xmlSecAssert2(outBuf != NULL, -1); if (!CryptEncrypt(hKey, 0, TRUE, 0, outBuf, &dwInLen, dwBufLen)) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, NULL, "CryptEncrypt", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return (-1); } /* The output of CryptEncrypt is in little-endian format, so we have to convert to * big-endian first. */ for(i = 0; i < outSize / 2; i++) { ch = outBuf[i]; outBuf[i] = outBuf[outSize - (i + 1)]; outBuf[outSize - (i + 1)] = ch; } } else { dwOutLen = inSize; /* The input of CryptDecrypt is expected to be little-endian, * so we have to convert from big-endian to little endian. */ inBuf = xmlSecBufferGetData(in); outBuf = xmlSecBufferGetData(out); xmlSecAssert2(inBuf != 0, -1); xmlSecAssert2(outBuf != 0, -1); for (i = 0; i < inSize; i++) { outBuf[i] = inBuf[inSize - (i + 1)]; } if (0 == (hKey = xmlSecMSCryptoKeyDataGetDecryptKey(ctx->data))) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, NULL, "xmlSecMSCryptoKeyDataGetKey", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return (-1); } if (!CryptDecrypt(hKey, 0, TRUE, 0, outBuf, &dwOutLen)) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "CryptDecrypt", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = dwOutLen; } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", inSize); return(-1); } return(0); }
static int xmlSecMSCryptoKWDes3Execute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) { xmlSecMSCryptoKWDes3CtxPtr ctx; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize, keySize; int ret; xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecMSCryptoTransformKWDes3Id), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecMSCryptoKWDes3Size), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecMSCryptoKWDes3GetCtx(transform); xmlSecAssert2(ctx != NULL, -1); keySize = xmlSecBufferGetSize(&(ctx->keyBuffer)); xmlSecAssert2(keySize == XMLSEC_KW_DES3_KEY_LENGTH, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); if(transform->status == xmlSecTransformStatusNone) { transform->status = xmlSecTransformStatusWorking; } if((transform->status == xmlSecTransformStatusWorking) && (last == 0)) { /* just do nothing */ } else if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) { if((inSize % XMLSEC_KW_DES3_BLOCK_LENGTH) != 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "%d bytes - not %d bytes aligned", inSize, XMLSEC_KW_DES3_BLOCK_LENGTH); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { /* the encoded key might be 16 bytes longer plus one block just in case */ outSize = inSize + XMLSEC_KW_DES3_IV_LENGTH + XMLSEC_KW_DES3_BLOCK_LENGTH + XMLSEC_KW_DES3_BLOCK_LENGTH; } else { /* just in case, add a block */ outSize = inSize + XMLSEC_KW_DES3_BLOCK_LENGTH; } ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { ret = xmlSecKWDes3Encode(&xmlSecMSCryptoKWDes3ImplKlass, ctx, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(out), outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecKWDes3Encode", XMLSEC_ERRORS_R_XMLSEC_FAILED, "key=%d,in=%d,out=%d", keySize, inSize, outSize); return(-1); } outSize = ret; } else { ret = xmlSecKWDes3Decode(&xmlSecMSCryptoKWDes3ImplKlass, ctx, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(out), outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecKWDes3Decode", XMLSEC_ERRORS_R_XMLSEC_FAILED, "key=%d,in=%d,out=%d", keySize, inSize, outSize); return(-1); } outSize = ret; } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", inSize); return(-1); } transform->status = xmlSecTransformStatusFinished; } else if(transform->status == xmlSecTransformStatusFinished) { /* the only way we can get here is if there is no input */ xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1); } else { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_STATUS, "status=%d", transform->status); return(-1); } return(0); }
static int xmlSecNssKWAesExecute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) { xmlSecBufferPtr in, out, key; xmlSecSize inSize, outSize, keySize, expectedKeySize; int ret; xmlSecAssert2(xmlSecNssKWAesCheckId(transform), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecNssKWAesSize), -1); xmlSecAssert2(transformCtx != NULL, -1); key = xmlSecNssKWAesGetKey(transform); xmlSecAssert2(key != NULL, -1); keySize = xmlSecBufferGetSize(key); expectedKeySize = xmlSecNssKWAesGetKeySize(transform); xmlSecAssert2(keySize == expectedKeySize, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); if(transform->status == xmlSecTransformStatusNone) { transform->status = xmlSecTransformStatusWorking; } if((transform->status == xmlSecTransformStatusWorking) && (last == 0)) { /* just do nothing */ } else if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) { if((inSize % 8) != 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "size=%d(not 8 bytes aligned)", inSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { /* the encoded key might be 8 bytes longer plus 8 bytes just in case */ outSize = inSize + XMLSEC_NSS_KW_AES_MAGIC_BLOCK_SIZE + XMLSEC_NSS_AES_BLOCK_SIZE; } else { outSize = inSize + XMLSEC_NSS_AES_BLOCK_SIZE; } ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "outSize=%d", outSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { ret = xmlSecNssKWAesOp(xmlSecBufferGetData(key), keySize, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(out), outSize, 1); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecNssKWAesOp", XMLSEC_ERRORS_R_XMLSEC_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = ret; } else { ret = xmlSecNssKWAesOp(xmlSecBufferGetData(key), keySize, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(out), outSize, 0); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecNssKWAesOp", XMLSEC_ERRORS_R_XMLSEC_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = ret; } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, g "outSize=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, "inSize%d", inSize); return(-1); } transform->status = xmlSecTransformStatusFinished; } else if(transform->status == xmlSecTransformStatusFinished) { /* the only way we can get here is if there is no input */ xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1); } else { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_STATUS, "status=%d", transform->status); return(-1); } return(0); }
static int xmlSecBase64Execute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) { xmlSecBase64CtxPtr ctx; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize, outLen; int ret; xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecTransformBase64Id), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncode) || (transform->operation == xmlSecTransformOperationDecode), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecBase64GetCtx(transform); xmlSecAssert2(ctx != NULL, -1); in = &(transform->inBuf); out = &(transform->outBuf); if(transform->status == xmlSecTransformStatusNone) { ctx->encode = (transform->operation == xmlSecTransformOperationEncode) ? 1 : 0; transform->status = xmlSecTransformStatusWorking; } switch(transform->status) { case xmlSecTransformStatusWorking: inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); if(inSize > 0) { if(ctx->encode != 0) { outLen = 4 * inSize / 3 + 8; if(ctx->columns > 0) { outLen += inSize / ctx->columns + 4; } } else { outLen = 3 * inSize / 4 + 8; } ret = xmlSecBufferSetMaxSize(out, outSize + outLen); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize + outLen); return(-1); } /* encode/decode the next chunk */ ret = xmlSecBase64CtxUpdate(ctx, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(out) + outSize, outLen); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBase64CtxUpdate", XMLSEC_ERRORS_R_XMLSEC_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outLen = ret; /* set correct size */ ret = xmlSecBufferSetSize(out, outSize + outLen); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize + outLen); return(-1); } /* remove chunk from input */ ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", inSize); return(-1); } } if(last) { outSize = xmlSecBufferGetSize(out); ret = xmlSecBufferSetMaxSize(out, outSize + 16); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize + 16); return(-1); } /* add from ctx buffer */ ret = xmlSecBase64CtxFinal(ctx, xmlSecBufferGetData(out) + outSize, 16); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBase64CtxFinal", XMLSEC_ERRORS_R_XMLSEC_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outLen = ret; /* set correct size */ ret = xmlSecBufferSetSize(out, outSize + outLen); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize + outLen); return(-1); } transform->status = xmlSecTransformStatusFinished; } break; case xmlSecTransformStatusFinished: /* the only way we can get here is if there is no input */ xmlSecAssert2(xmlSecBufferGetSize(in) == 0, -1); break; default: xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_STATUS, "status=%d", transform->status); return(-1); } return(0); }
static int xmlSecNssKWAesExecute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) { xmlSecNssKWAesCtxPtr ctx; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize, keySize; int ret; xmlSecAssert2(xmlSecNssKWAesCheckId(transform), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecNssKWAesSize), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecNssKWAesGetCtx(transform); xmlSecAssert2(ctx != NULL, -1); keySize = xmlSecBufferGetSize(&(ctx->keyBuffer)); xmlSecAssert2(keySize == ctx->keyExpectedSize, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); if(transform->status == xmlSecTransformStatusNone) { transform->status = xmlSecTransformStatusWorking; } if((transform->status == xmlSecTransformStatusWorking) && (last == 0)) { /* just do nothing */ } else if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) { if((inSize % 8) != 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "size=%d(not 8 bytes aligned)", inSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { /* the encoded key might be 8 bytes longer plus 8 bytes just in case */ outSize = inSize + XMLSEC_KW_AES_MAGIC_BLOCK_SIZE + XMLSEC_KW_AES_BLOCK_SIZE; } else { outSize = inSize + XMLSEC_KW_AES_BLOCK_SIZE; } ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "outSize=%d", outSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { PK11SymKey *aeskey = NULL; /* create key */ aeskey = xmlSecNssMakeAesKey(xmlSecBufferGetData(&(ctx->keyBuffer)), keySize, 1); /* encrypt */ if(aeskey == NULL) { xmlSecError(XMLSEC_ERRORS_HERE, NULL, "xmlSecNssMakeAesKey", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } /* encrypt */ ret = xmlSecKWAesEncode(&xmlSecNssKWAesKlass, aeskey, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(out), outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecKWAesEncode", XMLSEC_ERRORS_R_XMLSEC_FAILED, XMLSEC_ERRORS_NO_MESSAGE); PK11_FreeSymKey(aeskey); return(-1); } outSize = ret; PK11_FreeSymKey(aeskey); } else { PK11SymKey *aeskey = NULL; /* create key */ aeskey = xmlSecNssMakeAesKey(xmlSecBufferGetData(&(ctx->keyBuffer)), keySize, 0); /* decrypt */ if(aeskey == NULL) { xmlSecError(XMLSEC_ERRORS_HERE, NULL, "xmlSecNssMakeAesKey", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } /* decrypt */ ret = xmlSecKWAesDecode(&xmlSecNssKWAesKlass, aeskey, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(out), outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecKWAesDecode", XMLSEC_ERRORS_R_XMLSEC_FAILED, XMLSEC_ERRORS_NO_MESSAGE); PK11_FreeSymKey(aeskey); return(-1); } outSize = ret; PK11_FreeSymKey(aeskey); } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "outSize=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, "inSize%d", inSize); return(-1); } transform->status = xmlSecTransformStatusFinished; } else if(transform->status == xmlSecTransformStatusFinished) { /* the only way we can get here is if there is no input */ xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1); } else { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_STATUS, "status=%d", transform->status); return(-1); } return(0); }
static int xmlSecOpenSSLEvpSignatureExecute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) { xmlSecOpenSSLEvpSignatureCtxPtr ctx; xmlSecBufferPtr in, out; xmlSecSize inSize; xmlSecSize outSize; int ret; xmlSecAssert2(xmlSecOpenSSLEvpSignatureCheckId(transform), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationSign) || (transform->operation == xmlSecTransformOperationVerify), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLEvpSignatureSize), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecOpenSSLEvpSignatureGetCtx(transform); xmlSecAssert2(ctx != NULL, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); ctx = xmlSecOpenSSLEvpSignatureGetCtx(transform); xmlSecAssert2(ctx != NULL, -1); xmlSecAssert2(ctx->digest != NULL, -1); xmlSecAssert2(ctx->pKey != NULL, -1); if(transform->status == xmlSecTransformStatusNone) { xmlSecAssert2(outSize == 0, -1); if(transform->operation == xmlSecTransformOperationSign) { #ifndef XMLSEC_OPENSSL_096 ret = EVP_SignInit(&(ctx->digestCtx), ctx->digest); if(ret != 1) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "EVP_SignInit", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } #else /* XMLSEC_OPENSSL_096 */ EVP_SignInit(&(ctx->digestCtx), ctx->digest); #endif /* XMLSEC_OPENSSL_096 */ } else { #ifndef XMLSEC_OPENSSL_096 ret = EVP_VerifyInit(&(ctx->digestCtx), ctx->digest); if(ret != 1) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "EVP_VerifyInit", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } #else /* XMLSEC_OPENSSL_096 */ EVP_VerifyInit(&(ctx->digestCtx), ctx->digest); #endif /* XMLSEC_OPENSSL_096 */ } transform->status = xmlSecTransformStatusWorking; } if((transform->status == xmlSecTransformStatusWorking) && (inSize > 0)) { xmlSecAssert2(outSize == 0, -1); if(transform->operation == xmlSecTransformOperationSign) { #ifndef XMLSEC_OPENSSL_096 ret = EVP_SignUpdate(&(ctx->digestCtx), xmlSecBufferGetData(in), inSize); if(ret != 1) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "EVP_SignUpdate", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } #else /* XMLSEC_OPENSSL_096 */ EVP_SignUpdate(&(ctx->digestCtx), xmlSecBufferGetData(in), inSize); #endif /* XMLSEC_OPENSSL_096 */ } else { #ifndef XMLSEC_OPENSSL_096 ret = EVP_VerifyUpdate(&(ctx->digestCtx), xmlSecBufferGetData(in), inSize); if(ret != 1) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "EVP_VerifyUpdate", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } #else /* XMLSEC_OPENSSL_096 */ EVP_VerifyUpdate(&(ctx->digestCtx), xmlSecBufferGetData(in), inSize); #endif /* XMLSEC_OPENSSL_096 */ } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } } if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) { xmlSecAssert2(outSize == 0, -1); if(transform->operation == xmlSecTransformOperationSign) { unsigned int signSize; /* this is a hack: for rsa signatures * we get size from EVP_PKEY_size(), * for dsa signature we use a fixed constant */ signSize = EVP_PKEY_size(ctx->pKey); #ifndef XMLSEC_NO_DSA if(signSize < XMLSEC_OPENSSL_DSA_SIGNATURE_SIZE) { signSize = XMLSEC_OPENSSL_DSA_SIGNATURE_SIZE; } #endif /* XMLSEC_NO_DSA */ ret = xmlSecBufferSetMaxSize(out, signSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%u", signSize); return(-1); } ret = EVP_SignFinal(&(ctx->digestCtx), xmlSecBufferGetData(out), &signSize, ctx->pKey); if(ret != 1) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "EVP_SignFinal", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } ret = xmlSecBufferSetSize(out, signSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%u", signSize); return(-1); } } transform->status = xmlSecTransformStatusFinished; } if((transform->status == xmlSecTransformStatusWorking) || (transform->status == xmlSecTransformStatusFinished)) { /* the only way we can get here is if there is no input */ xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1); } else { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_STATUS, "status=%d", transform->status); return(-1); } return(0); }
static int xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) { xmlSecOpenSSLRsaOaepCtxPtr ctx; xmlSecSize paramsSize; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize; xmlSecSize keySize; int ret; xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecOpenSSLRsaOaepGetCtx(transform); xmlSecAssert2(ctx != NULL, -1); xmlSecAssert2(ctx->pKey != NULL, -1); xmlSecAssert2(ctx->pKey->type == EVP_PKEY_RSA, -1); xmlSecAssert2(ctx->pKey->pkey.rsa != NULL, -1); keySize = RSA_size(ctx->pKey->pkey.rsa); xmlSecAssert2(keySize > 0, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); /* the encoded size is equal to the keys size so we could not * process more than that */ if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "%d when expected less than %d", inSize, keySize); return(-1); } else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "%d when expected %d", inSize, keySize); return(-1); } outSize = keySize; ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } paramsSize = xmlSecBufferGetSize(&(ctx->oaepParams)); if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize == 0)) { /* encode w/o OAEPParams --> simple */ ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING); if(ret <= 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_public_encrypt(RSA_PKCS1_OAEP_PADDING)", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = ret; } else if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize > 0)) { xmlSecAssert2(xmlSecBufferGetData(&(ctx->oaepParams)) != NULL, -1); /* add space for padding */ ret = xmlSecBufferSetMaxSize(in, keySize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", keySize); return(-1); } /* add padding */ ret = RSA_padding_add_PKCS1_OAEP(xmlSecBufferGetData(in), keySize, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(&(ctx->oaepParams)), paramsSize); if(ret != 1) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_padding_add_PKCS1_OAEP", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } inSize = keySize; /* encode with OAEPParams */ ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), ctx->pKey->pkey.rsa, RSA_NO_PADDING); if(ret <= 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_public_encrypt(RSA_NO_PADDING)", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = ret; } else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize == 0)) { ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING); if(ret <= 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_private_decrypt(RSA_PKCS1_OAEP_PADDING)", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = ret; } else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize != 0)) { BIGNUM bn; ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), ctx->pKey->pkey.rsa, RSA_NO_PADDING); if(ret <= 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_private_decrypt(RSA_NO_PADDING)", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = ret; /* * the private decrypt w/o padding adds '0's at the begginning. * it's not clear for me can I simply skip all '0's from the * beggining so I have to do decode it back to BIGNUM and dump * buffer again */ BN_init(&bn); if(BN_bin2bn(xmlSecBufferGetData(out), outSize, &bn) == NULL) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "BN_bin2bn", XMLSEC_ERRORS_R_CRYPTO_FAILED, "size=%d", outSize); BN_clear_free(&bn); return(-1); } ret = BN_bn2bin(&bn, xmlSecBufferGetData(out)); if(ret <= 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "BN_bn2bin", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); BN_clear_free(&bn); return(-1); } BN_clear_free(&bn); outSize = ret; ret = RSA_padding_check_PKCS1_OAEP(xmlSecBufferGetData(out), outSize, xmlSecBufferGetData(out), outSize, keySize, xmlSecBufferGetData(&(ctx->oaepParams)), paramsSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_padding_check_PKCS1_OAEP", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = ret; } else { xmlSecAssert2("we could not be here" == NULL, -1); return(-1); } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", inSize); return(-1); } return(0); }
static int xmlSecOpenSSLRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) { xmlSecOpenSSLRsaPkcs1CtxPtr ctx; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize; xmlSecSize keySize; int ret; xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaPkcs1Size), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform); xmlSecAssert2(ctx != NULL, -1); xmlSecAssert2(ctx->pKey != NULL, -1); xmlSecAssert2(ctx->pKey->type == EVP_PKEY_RSA, -1); xmlSecAssert2(ctx->pKey->pkey.rsa != NULL, -1); keySize = RSA_size(ctx->pKey->pkey.rsa); xmlSecAssert2(keySize > 0, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); /* the encoded size is equal to the keys size so we could not * process more than that */ if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "%d when expected less than %d", inSize, keySize); return(-1); } else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "%d when expected %d", inSize, keySize); return(-1); } outSize = keySize; ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), ctx->pKey->pkey.rsa, RSA_PKCS1_PADDING); if(ret <= 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_public_encrypt", XMLSEC_ERRORS_R_CRYPTO_FAILED, "size=%d", inSize); return(-1); } outSize = ret; } else { ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), ctx->pKey->pkey.rsa, RSA_PKCS1_PADDING); if(ret <= 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_private_decrypt", XMLSEC_ERRORS_R_CRYPTO_FAILED, "size=%d", inSize); return(-1); } outSize = ret; } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", inSize); return(-1); } return(0); }
static int xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) { xmlSecOpenSSLRsaOaepCtxPtr ctx; xmlSecSize paramsSize; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize; xmlSecSize keySize; RSA *rsa; int ret; xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecOpenSSLRsaOaepGetCtx(transform); xmlSecAssert2(ctx != NULL, -1); xmlSecAssert2(ctx->pKey != NULL, -1); xmlSecAssert2(EVP_PKEY_base_id(ctx->pKey) == EVP_PKEY_RSA, -1); rsa = EVP_PKEY_get0_RSA(ctx->pKey); xmlSecAssert2(rsa != NULL, -1); keySize = RSA_size(rsa); xmlSecAssert2(keySize > 0, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); /* the encoded size is equal to the keys size so we could not * process more than that */ if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) { xmlSecInvalidSizeLessThanError("Input data", inSize, keySize, xmlSecTransformGetName(transform)); return(-1); } else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) { xmlSecInvalidSizeError("Input data", inSize, keySize, xmlSecTransformGetName(transform)); return(-1); } outSize = keySize; ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecInternalError2("xmlSecBufferSetMaxSize", xmlSecTransformGetName(transform), "size=%d", outSize); return(-1); } paramsSize = xmlSecBufferGetSize(&(ctx->oaepParams)); if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize == 0)) { /* encode w/o OAEPParams --> simple */ ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), rsa, RSA_PKCS1_OAEP_PADDING); if(ret <= 0) { xmlSecOpenSSLError("RSA_public_encrypt(RSA_PKCS1_OAEP_PADDING)", xmlSecTransformGetName(transform)); return(-1); } outSize = ret; } else if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize > 0)) { xmlSecAssert2(xmlSecBufferGetData(&(ctx->oaepParams)) != NULL, -1); /* add space for padding */ ret = xmlSecBufferSetMaxSize(in, keySize); if(ret < 0) { xmlSecInternalError2("xmlSecBufferSetMaxSize", xmlSecTransformGetName(transform), "size=%d", keySize); return(-1); } /* add padding */ ret = RSA_padding_add_PKCS1_OAEP(xmlSecBufferGetData(in), keySize, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(&(ctx->oaepParams)), paramsSize); if(ret != 1) { xmlSecOpenSSLError("RSA_padding_add_PKCS1_OAEP", xmlSecTransformGetName(transform)); return(-1); } inSize = keySize; /* encode with OAEPParams */ ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), rsa, RSA_NO_PADDING); if(ret <= 0) { xmlSecOpenSSLError("RSA_public_encrypt(RSA_NO_PADDING)", xmlSecTransformGetName(transform)); return(-1); } outSize = ret; } else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize == 0)) { ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), rsa, RSA_PKCS1_OAEP_PADDING); if(ret <= 0) { xmlSecOpenSSLError("RSA_private_decrypt(RSA_PKCS1_OAEP_PADDING)", xmlSecTransformGetName(transform)); return(-1); } outSize = ret; } else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize != 0)) { BIGNUM * bn; bn = BN_new(); if(bn == NULL) { xmlSecOpenSSLError("BN_new()", xmlSecTransformGetName(transform)); return(-1); } ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), rsa, RSA_NO_PADDING); if(ret <= 0) { xmlSecOpenSSLError("RSA_private_decrypt(RSA_NO_PADDING)", xmlSecTransformGetName(transform)); BN_free(bn); return(-1); } outSize = ret; /* * the private decrypt w/o padding adds '0's at the begginning. * it's not clear for me can I simply skip all '0's from the * beggining so I have to do decode it back to BIGNUM and dump * buffer again */ if(BN_bin2bn(xmlSecBufferGetData(out), outSize, bn) == NULL) { xmlSecOpenSSLError2("BN_bin2bn", xmlSecTransformGetName(transform), "size=%lu", (unsigned long)outSize); BN_free(bn); return(-1); } ret = BN_bn2bin(bn, xmlSecBufferGetData(out)); if(ret <= 0) { xmlSecOpenSSLError("BN_bn2bin", xmlSecTransformGetName(transform)); BN_free(bn); return(-1); } BN_free(bn); outSize = ret; ret = RSA_padding_check_PKCS1_OAEP(xmlSecBufferGetData(out), outSize, xmlSecBufferGetData(out), outSize, keySize, xmlSecBufferGetData(&(ctx->oaepParams)), paramsSize); if(ret < 0) { xmlSecOpenSSLError("RSA_padding_check_PKCS1_OAEP", xmlSecTransformGetName(transform)); return(-1); } outSize = ret; } else { xmlSecOtherError3(XMLSEC_ERRORS_R_INVALID_OPERATION, xmlSecTransformGetName(transform), "Unexpected transform operation: %ld; paramsSize: %ld", (long int)transform->operation, (long int)paramsSize); return(-1); } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecInternalError2("xmlSecBufferSetSize", xmlSecTransformGetName(transform), "size=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecInternalError2("xmlSecBufferRemoveHead", xmlSecTransformGetName(transform), "size=%d", inSize); return(-1); } return(0); }
static int xmlSecOpenSSLRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) { xmlSecOpenSSLRsaPkcs1CtxPtr ctx; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize; xmlSecSize keySize; RSA *rsa; int ret; xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaPkcs1Size), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform); xmlSecAssert2(ctx != NULL, -1); xmlSecAssert2(ctx->pKey != NULL, -1); xmlSecAssert2(EVP_PKEY_base_id(ctx->pKey) == EVP_PKEY_RSA, -1); rsa = EVP_PKEY_get0_RSA(ctx->pKey); xmlSecAssert2(rsa != NULL, -1); keySize = RSA_size(rsa); xmlSecAssert2(keySize > 0, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); /* the encoded size is equal to the keys size so we could not * process more than that */ if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) { xmlSecInvalidSizeLessThanError("Input data", inSize, keySize, xmlSecTransformGetName(transform)); return(-1); } else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) { xmlSecInvalidSizeError("Input data", inSize, keySize, xmlSecTransformGetName(transform)); return(-1); } outSize = keySize; ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecInternalError2("xmlSecBufferSetMaxSize", xmlSecTransformGetName(transform), "size=%d", outSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), rsa, RSA_PKCS1_PADDING); if(ret <= 0) { xmlSecOpenSSLError2("RSA_public_encrypt", xmlSecTransformGetName(transform), "size=%lu", (unsigned long)inSize); return(-1); } outSize = ret; } else { ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), rsa, RSA_PKCS1_PADDING); if(ret <= 0) { xmlSecOpenSSLError2("RSA_private_decrypt", xmlSecTransformGetName(transform), "size=%lu", (unsigned long)inSize); return(-1); } outSize = ret; } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecInternalError2("xmlSecBufferSetSize", xmlSecTransformGetName(transform), "size=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecInternalError2("xmlSecBufferRemoveHead", xmlSecTransformGetName(transform), "size=%d", inSize); return(-1); } return(0); }