PyObject *xmlsec_BufferSetMaxSize(PyObject *self, PyObject *args) {
PyObject *buf_obj;
xmlSecBufferPtr buf;
xmlSecSize size;
if (CheckArgs(args, "OI:bufferSetMaxSize")) {
if (!PyArg_ParseTuple(args, "Oi:bufferSetMaxSize", &buf_obj, &size))
return NULL;
}
else return NULL;
buf = xmlSecBufferPtr_get(buf_obj);
return (wrap_int(xmlSecBufferSetMaxSize(buf, size)));
}
static int
xmlSecMSCryptoRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) {
xmlSecMSCryptoRsaPkcs1CtxPtr ctx;
xmlSecBufferPtr in, out;
xmlSecSize inSize, outSize;
xmlSecSize keySize;
int ret;
HCRYPTKEY hKey = 0;
DWORD dwInLen;
DWORD dwBufLen;
DWORD dwOutLen;
BYTE * outBuf;
BYTE * inBuf;
int i;
xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecMSCryptoTransformRsaPkcs1Id), -1);
xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecMSCryptoRsaPkcs1Size), -1);
xmlSecAssert2(transformCtx != NULL, -1);
ctx = xmlSecMSCryptoRsaPkcs1GetCtx(transform);
xmlSecAssert2(ctx != NULL, -1);
xmlSecAssert2(ctx->data != NULL, -1);
keySize = xmlSecKeyDataGetSize(ctx->data) / 8;
xmlSecAssert2(keySize > 0, -1);
in = &(transform->inBuf);
out = &(transform->outBuf);
inSize = xmlSecBufferGetSize(in);
outSize = xmlSecBufferGetSize(out);
xmlSecAssert2(outSize == 0, -1);
/* the encoded size is equal to the keys size so we could not
* process more than that */
if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) {
xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_SIZE,
"%d when expected less than %d", inSize, keySize);
return(-1);
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) {
xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_SIZE,
"%d when expected %d", inSize, keySize);
return(-1);
}
outSize = keySize;
ret = xmlSecBufferSetMaxSize(out, outSize);
if(ret < 0) {
xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetMaxSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", outSize);
return(-1);
}
if(transform->operation == xmlSecTransformOperationEncrypt) {
BYTE ch;
if(inSize > outSize) {
xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_SIZE,
"inSize=%d;outSize=%d",
inSize, outSize);
return(-1);
}
ret = xmlSecBufferSetData(out, xmlSecBufferGetData(in), inSize);
if(ret < 0) {
xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetData",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", inSize);
return(-1);
}
dwInLen = inSize;
dwBufLen = outSize;
if (0 == (hKey = xmlSecMSCryptoKeyDataGetKey(ctx->data, xmlSecKeyDataTypePublic))) {
xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE,
NULL,
"xmlSecMSCryptoKeyDataGetKey",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return (-1);
}
outBuf = xmlSecBufferGetData(out);
xmlSecAssert2(outBuf != NULL, -1);
if (!CryptEncrypt(hKey, 0, TRUE, 0, outBuf, &dwInLen, dwBufLen)) {
xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE,
NULL,
"CryptEncrypt",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return (-1);
}
/* The output of CryptEncrypt is in little-endian format, so we have to convert to
* big-endian first.
*/
for(i = 0; i < outSize / 2; i++) {
ch = outBuf[i];
outBuf[i] = outBuf[outSize - (i + 1)];
outBuf[outSize - (i + 1)] = ch;
}
} else {
dwOutLen = inSize;
/* The input of CryptDecrypt is expected to be little-endian,
* so we have to convert from big-endian to little endian.
*/
inBuf = xmlSecBufferGetData(in);
outBuf = xmlSecBufferGetData(out);
xmlSecAssert2(inBuf != 0, -1);
xmlSecAssert2(outBuf != 0, -1);
for (i = 0; i < inSize; i++) {
outBuf[i] = inBuf[inSize - (i + 1)];
}
if (0 == (hKey = xmlSecMSCryptoKeyDataGetDecryptKey(ctx->data))) {
xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE,
NULL,
"xmlSecMSCryptoKeyDataGetKey",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return (-1);
}
if (!CryptDecrypt(hKey, 0, TRUE, 0, outBuf, &dwOutLen)) {
xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"CryptDecrypt",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
outSize = dwOutLen;
}
ret = xmlSecBufferSetSize(out, outSize);
if(ret < 0) {
xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", outSize);
return(-1);
}
ret = xmlSecBufferRemoveHead(in, inSize);
if(ret < 0) {
xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferRemoveHead",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", inSize);
return(-1);
}
return(0);
}
static int
xmlSecMSCryptoKWDes3Execute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) {
xmlSecMSCryptoKWDes3CtxPtr ctx;
xmlSecBufferPtr in, out;
xmlSecSize inSize, outSize, keySize;
int ret;
xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecMSCryptoTransformKWDes3Id), -1);
xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecMSCryptoKWDes3Size), -1);
xmlSecAssert2(transformCtx != NULL, -1);
ctx = xmlSecMSCryptoKWDes3GetCtx(transform);
xmlSecAssert2(ctx != NULL, -1);
keySize = xmlSecBufferGetSize(&(ctx->keyBuffer));
xmlSecAssert2(keySize == XMLSEC_KW_DES3_KEY_LENGTH, -1);
in = &(transform->inBuf);
out = &(transform->outBuf);
inSize = xmlSecBufferGetSize(in);
outSize = xmlSecBufferGetSize(out);
xmlSecAssert2(outSize == 0, -1);
if(transform->status == xmlSecTransformStatusNone) {
transform->status = xmlSecTransformStatusWorking;
}
if((transform->status == xmlSecTransformStatusWorking) && (last == 0)) {
/* just do nothing */
} else if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) {
if((inSize % XMLSEC_KW_DES3_BLOCK_LENGTH) != 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_SIZE,
"%d bytes - not %d bytes aligned",
inSize, XMLSEC_KW_DES3_BLOCK_LENGTH);
return(-1);
}
if(transform->operation == xmlSecTransformOperationEncrypt) {
/* the encoded key might be 16 bytes longer plus one block just in case */
outSize = inSize + XMLSEC_KW_DES3_IV_LENGTH +
XMLSEC_KW_DES3_BLOCK_LENGTH +
XMLSEC_KW_DES3_BLOCK_LENGTH;
} else {
/* just in case, add a block */
outSize = inSize + XMLSEC_KW_DES3_BLOCK_LENGTH;
}
ret = xmlSecBufferSetMaxSize(out, outSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetMaxSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", outSize);
return(-1);
}
if(transform->operation == xmlSecTransformOperationEncrypt) {
ret = xmlSecKWDes3Encode(&xmlSecMSCryptoKWDes3ImplKlass, ctx,
xmlSecBufferGetData(in), inSize,
xmlSecBufferGetData(out), outSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecKWDes3Encode",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"key=%d,in=%d,out=%d",
keySize, inSize, outSize);
return(-1);
}
outSize = ret;
} else {
ret = xmlSecKWDes3Decode(&xmlSecMSCryptoKWDes3ImplKlass, ctx,
xmlSecBufferGetData(in), inSize,
xmlSecBufferGetData(out), outSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecKWDes3Decode",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"key=%d,in=%d,out=%d",
keySize, inSize, outSize);
return(-1);
}
outSize = ret;
}
ret = xmlSecBufferSetSize(out, outSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", outSize);
return(-1);
}
ret = xmlSecBufferRemoveHead(in, inSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferRemoveHead",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", inSize);
return(-1);
}
transform->status = xmlSecTransformStatusFinished;
} else if(transform->status == xmlSecTransformStatusFinished) {
/* the only way we can get here is if there is no input */
xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1);
} else {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_STATUS,
"status=%d", transform->status);
return(-1);
}
return(0);
}
static int
xmlSecBase64Execute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) {
xmlSecBase64CtxPtr ctx;
xmlSecBufferPtr in, out;
xmlSecSize inSize, outSize, outLen;
int ret;
xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecTransformBase64Id), -1);
xmlSecAssert2((transform->operation == xmlSecTransformOperationEncode) || (transform->operation == xmlSecTransformOperationDecode), -1);
xmlSecAssert2(transformCtx != NULL, -1);
ctx = xmlSecBase64GetCtx(transform);
xmlSecAssert2(ctx != NULL, -1);
in = &(transform->inBuf);
out = &(transform->outBuf);
if(transform->status == xmlSecTransformStatusNone) {
ctx->encode = (transform->operation == xmlSecTransformOperationEncode) ? 1 : 0;
transform->status = xmlSecTransformStatusWorking;
}
switch(transform->status) {
case xmlSecTransformStatusWorking:
inSize = xmlSecBufferGetSize(in);
outSize = xmlSecBufferGetSize(out);
if(inSize > 0) {
if(ctx->encode != 0) {
outLen = 4 * inSize / 3 + 8;
if(ctx->columns > 0) {
outLen += inSize / ctx->columns + 4;
}
} else {
outLen = 3 * inSize / 4 + 8;
}
ret = xmlSecBufferSetMaxSize(out, outSize + outLen);
if(ret < 0) {
xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetMaxSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", outSize + outLen);
return(-1);
}
/* encode/decode the next chunk */
ret = xmlSecBase64CtxUpdate(ctx, xmlSecBufferGetData(in), inSize,
xmlSecBufferGetData(out) + outSize,
outLen);
if(ret < 0) {
xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBase64CtxUpdate",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
outLen = ret;
/* set correct size */
ret = xmlSecBufferSetSize(out, outSize + outLen);
if(ret < 0) {
xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", outSize + outLen);
return(-1);
}
/* remove chunk from input */
ret = xmlSecBufferRemoveHead(in, inSize);
if(ret < 0) {
xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferRemoveHead",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", inSize);
return(-1);
}
}
if(last) {
outSize = xmlSecBufferGetSize(out);
ret = xmlSecBufferSetMaxSize(out, outSize + 16);
if(ret < 0) {
xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetMaxSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", outSize + 16);
return(-1);
}
/* add from ctx buffer */
ret = xmlSecBase64CtxFinal(ctx, xmlSecBufferGetData(out) + outSize, 16);
if(ret < 0) {
xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBase64CtxFinal",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
outLen = ret;
/* set correct size */
ret = xmlSecBufferSetSize(out, outSize + outLen);
if(ret < 0) {
xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", outSize + outLen);
return(-1);
}
transform->status = xmlSecTransformStatusFinished;
}
break;
case xmlSecTransformStatusFinished:
/* the only way we can get here is if there is no input */
xmlSecAssert2(xmlSecBufferGetSize(in) == 0, -1);
break;
default:
xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_STATUS,
"status=%d", transform->status);
return(-1);
}
return(0);
}
static int
xmlSecNssKWAesExecute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) {
xmlSecBufferPtr in, out, key;
xmlSecSize inSize, outSize, keySize, expectedKeySize;
int ret;
xmlSecAssert2(xmlSecNssKWAesCheckId(transform), -1);
xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecNssKWAesSize), -1);
xmlSecAssert2(transformCtx != NULL, -1);
key = xmlSecNssKWAesGetKey(transform);
xmlSecAssert2(key != NULL, -1);
keySize = xmlSecBufferGetSize(key);
expectedKeySize = xmlSecNssKWAesGetKeySize(transform);
xmlSecAssert2(keySize == expectedKeySize, -1);
in = &(transform->inBuf);
out = &(transform->outBuf);
inSize = xmlSecBufferGetSize(in);
outSize = xmlSecBufferGetSize(out);
xmlSecAssert2(outSize == 0, -1);
if(transform->status == xmlSecTransformStatusNone) {
transform->status = xmlSecTransformStatusWorking;
}
if((transform->status == xmlSecTransformStatusWorking) && (last == 0)) {
/* just do nothing */
} else if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) {
if((inSize % 8) != 0) {
xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_SIZE,
"size=%d(not 8 bytes aligned)", inSize);
return(-1);
}
if(transform->operation == xmlSecTransformOperationEncrypt) {
/* the encoded key might be 8 bytes longer plus 8 bytes just in case */
outSize = inSize + XMLSEC_NSS_KW_AES_MAGIC_BLOCK_SIZE +
XMLSEC_NSS_AES_BLOCK_SIZE;
} else {
outSize = inSize + XMLSEC_NSS_AES_BLOCK_SIZE;
}
ret = xmlSecBufferSetMaxSize(out, outSize);
if(ret < 0) {
xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetMaxSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"outSize=%d", outSize);
return(-1);
}
if(transform->operation == xmlSecTransformOperationEncrypt) {
ret = xmlSecNssKWAesOp(xmlSecBufferGetData(key), keySize,
xmlSecBufferGetData(in), inSize,
xmlSecBufferGetData(out), outSize, 1);
if(ret < 0) {
xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecNssKWAesOp",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
outSize = ret;
} else {
ret = xmlSecNssKWAesOp(xmlSecBufferGetData(key), keySize,
xmlSecBufferGetData(in), inSize,
xmlSecBufferGetData(out), outSize, 0);
if(ret < 0) {
xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecNssKWAesOp",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
outSize = ret;
}
ret = xmlSecBufferSetSize(out, outSize);
if(ret < 0) {
xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
g "outSize=%d", outSize);
return(-1);
}
ret = xmlSecBufferRemoveHead(in, inSize);
if(ret < 0) {
xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferRemoveHead",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"inSize%d", inSize);
return(-1);
}
transform->status = xmlSecTransformStatusFinished;
} else if(transform->status == xmlSecTransformStatusFinished) {
/* the only way we can get here is if there is no input */
xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1);
} else {
xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_STATUS,
"status=%d", transform->status);
return(-1);
}
return(0);
}
/**
* xmlSecBnFromString:
* @bn: the pointer to BN.
* @str: the string with BN.
* @base: the base for @str.
*
* Reads @bn from string @str assuming it has base @base.
*
* Returns 0 on success or a negative value if an error occurs.
*/
EXPORT_C
int
xmlSecBnFromString(xmlSecBnPtr bn, const xmlChar* str, xmlSecSize base) {
xmlSecSize i, len, size;
xmlSecByte ch;
xmlSecByte* data;
int positive;
int nn;
int ret;
xmlSecAssert2(bn != NULL, -1);
xmlSecAssert2(str != NULL, -1);
xmlSecAssert2(base > 1, -1);
xmlSecAssert2(base <= sizeof(xmlSecBnRevLookupTable), -1);
/* trivial case */
len = xmlStrlen(str);
if(len == 0) {
return(0);
}
/* The result size could not exceed the input string length
* because each char fits inside a byte in all cases :)
* In truth, it would be likely less than 1/2 input string length
* because each byte is represented by 2 chars. If needed,
* buffer size would be increased by Mul/Add functions.
* Finally, we can add one byte for 00 or 10 prefix.
*/
ret = xmlSecBufferSetMaxSize(bn, xmlSecBufferGetSize(bn) + len / 2 + 1 + 1);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
NULL,
"xmlSecBnRevLookupTable",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", len / 2 + 1);
return (-1);
}
/* figure out if it is positive or negative number */
positive = 1;
i = 0;
while(i < len) {
ch = str[i++];
/* skip spaces */
if(isspace(ch)) {
continue;
}
/* check if it is + or - */
if(ch == '+') {
positive = 1;
break;
} else if(ch == '-') {
positive = 0;
break;
}
/* otherwise, it must be start of the number */
nn = xmlSecBnLookupTable[ch];
if((nn >= 0) && ((xmlSecSize)nn < base)) {
xmlSecAssert2(i > 0, -1);
/* no sign, positive by default */
positive = 1;
--i; /* make sure that we will look at this character in next loop */
break;
} else {
xmlSecError(XMLSEC_ERRORS_HERE,
NULL,
NULL,
XMLSEC_ERRORS_R_INVALID_DATA,
"char=%c;base=%d",
ch, base);
return (-1);
}
}
/* now parse the number itself */
while(i < len) {
ch = str[i++];
if(isspace(ch)) {
continue;
}
xmlSecAssert2(ch <(sizeof(xmlSecBnLookupTable)/sizeof(xmlSecBnLookupTable[0])), -1);
nn = xmlSecBnLookupTable[ch];
if((nn < 0) || ((xmlSecSize)nn > base)) {
xmlSecError(XMLSEC_ERRORS_HERE,
NULL,
NULL,
XMLSEC_ERRORS_R_INVALID_DATA,
"char=%c;base=%d",
ch, base);
return (-1);
}
ret = xmlSecBnMul(bn, base);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
NULL,
"xmlSecBnMul",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"base=%d", base);
return (-1);
}
ret = xmlSecBnAdd(bn, nn);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
NULL,
"xmlSecBnAdd",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"base=%d", base);
return (-1);
}
}
/* check if we need to add 00 prefix, do this for empty bn too */
data = xmlSecBufferGetData(bn);
size = xmlSecBufferGetSize(bn);
if(((size > 0) && (data[0] > 127)) || (size == 0)) {
ch = 0;
ret = xmlSecBufferPrepend(bn, &ch, 1);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
NULL,
"xmlSecBufferPrepend",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"base=%d", base);
return (-1);
}
}
/* do 2's compliment and add 1 to represent negative value */
if(positive == 0) {
data = xmlSecBufferGetData(bn);
size = xmlSecBufferGetSize(bn);
for(i = 0; i < size; ++i) {
data[i] ^= 0xFF;
}
ret = xmlSecBnAdd(bn, 1);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
NULL,
"xmlSecBnAdd",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"base=%d", base);
return (-1);
}
}
return(0);
}
static int
xmlSecNssKWAesExecute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) {
xmlSecNssKWAesCtxPtr ctx;
xmlSecBufferPtr in, out;
xmlSecSize inSize, outSize, keySize;
int ret;
xmlSecAssert2(xmlSecNssKWAesCheckId(transform), -1);
xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecNssKWAesSize), -1);
xmlSecAssert2(transformCtx != NULL, -1);
ctx = xmlSecNssKWAesGetCtx(transform);
xmlSecAssert2(ctx != NULL, -1);
keySize = xmlSecBufferGetSize(&(ctx->keyBuffer));
xmlSecAssert2(keySize == ctx->keyExpectedSize, -1);
in = &(transform->inBuf);
out = &(transform->outBuf);
inSize = xmlSecBufferGetSize(in);
outSize = xmlSecBufferGetSize(out);
xmlSecAssert2(outSize == 0, -1);
if(transform->status == xmlSecTransformStatusNone) {
transform->status = xmlSecTransformStatusWorking;
}
if((transform->status == xmlSecTransformStatusWorking) && (last == 0)) {
/* just do nothing */
} else if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) {
if((inSize % 8) != 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_SIZE,
"size=%d(not 8 bytes aligned)", inSize);
return(-1);
}
if(transform->operation == xmlSecTransformOperationEncrypt) {
/* the encoded key might be 8 bytes longer plus 8 bytes just in case */
outSize = inSize + XMLSEC_KW_AES_MAGIC_BLOCK_SIZE +
XMLSEC_KW_AES_BLOCK_SIZE;
} else {
outSize = inSize + XMLSEC_KW_AES_BLOCK_SIZE;
}
ret = xmlSecBufferSetMaxSize(out, outSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetMaxSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"outSize=%d", outSize);
return(-1);
}
if(transform->operation == xmlSecTransformOperationEncrypt) {
PK11SymKey *aeskey = NULL;
/* create key */
aeskey = xmlSecNssMakeAesKey(xmlSecBufferGetData(&(ctx->keyBuffer)), keySize, 1); /* encrypt */
if(aeskey == NULL) {
xmlSecError(XMLSEC_ERRORS_HERE,
NULL,
"xmlSecNssMakeAesKey",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
/* encrypt */
ret = xmlSecKWAesEncode(&xmlSecNssKWAesKlass, aeskey,
xmlSecBufferGetData(in), inSize,
xmlSecBufferGetData(out), outSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecKWAesEncode",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
PK11_FreeSymKey(aeskey);
return(-1);
}
outSize = ret;
PK11_FreeSymKey(aeskey);
} else {
PK11SymKey *aeskey = NULL;
/* create key */
aeskey = xmlSecNssMakeAesKey(xmlSecBufferGetData(&(ctx->keyBuffer)), keySize, 0); /* decrypt */
if(aeskey == NULL) {
xmlSecError(XMLSEC_ERRORS_HERE,
NULL,
"xmlSecNssMakeAesKey",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
/* decrypt */
ret = xmlSecKWAesDecode(&xmlSecNssKWAesKlass, aeskey,
xmlSecBufferGetData(in), inSize,
xmlSecBufferGetData(out), outSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecKWAesDecode",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
PK11_FreeSymKey(aeskey);
return(-1);
}
outSize = ret;
PK11_FreeSymKey(aeskey);
}
ret = xmlSecBufferSetSize(out, outSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"outSize=%d", outSize);
return(-1);
}
ret = xmlSecBufferRemoveHead(in, inSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferRemoveHead",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"inSize%d", inSize);
return(-1);
}
transform->status = xmlSecTransformStatusFinished;
} else if(transform->status == xmlSecTransformStatusFinished) {
/* the only way we can get here is if there is no input */
xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1);
} else {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_STATUS,
"status=%d", transform->status);
return(-1);
}
return(0);
}
static int
xmlSecOpenSSLEvpSignatureExecute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) {
xmlSecOpenSSLEvpSignatureCtxPtr ctx;
xmlSecBufferPtr in, out;
xmlSecSize inSize;
xmlSecSize outSize;
int ret;
xmlSecAssert2(xmlSecOpenSSLEvpSignatureCheckId(transform), -1);
xmlSecAssert2((transform->operation == xmlSecTransformOperationSign) || (transform->operation == xmlSecTransformOperationVerify), -1);
xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLEvpSignatureSize), -1);
xmlSecAssert2(transformCtx != NULL, -1);
ctx = xmlSecOpenSSLEvpSignatureGetCtx(transform);
xmlSecAssert2(ctx != NULL, -1);
in = &(transform->inBuf);
out = &(transform->outBuf);
inSize = xmlSecBufferGetSize(in);
outSize = xmlSecBufferGetSize(out);
ctx = xmlSecOpenSSLEvpSignatureGetCtx(transform);
xmlSecAssert2(ctx != NULL, -1);
xmlSecAssert2(ctx->digest != NULL, -1);
xmlSecAssert2(ctx->pKey != NULL, -1);
if(transform->status == xmlSecTransformStatusNone) {
xmlSecAssert2(outSize == 0, -1);
if(transform->operation == xmlSecTransformOperationSign) {
#ifndef XMLSEC_OPENSSL_096
ret = EVP_SignInit(&(ctx->digestCtx), ctx->digest);
if(ret != 1) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"EVP_SignInit",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
#else /* XMLSEC_OPENSSL_096 */
EVP_SignInit(&(ctx->digestCtx), ctx->digest);
#endif /* XMLSEC_OPENSSL_096 */
} else {
#ifndef XMLSEC_OPENSSL_096
ret = EVP_VerifyInit(&(ctx->digestCtx), ctx->digest);
if(ret != 1) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"EVP_VerifyInit",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
#else /* XMLSEC_OPENSSL_096 */
EVP_VerifyInit(&(ctx->digestCtx), ctx->digest);
#endif /* XMLSEC_OPENSSL_096 */
}
transform->status = xmlSecTransformStatusWorking;
}
if((transform->status == xmlSecTransformStatusWorking) && (inSize > 0)) {
xmlSecAssert2(outSize == 0, -1);
if(transform->operation == xmlSecTransformOperationSign) {
#ifndef XMLSEC_OPENSSL_096
ret = EVP_SignUpdate(&(ctx->digestCtx), xmlSecBufferGetData(in), inSize);
if(ret != 1) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"EVP_SignUpdate",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
#else /* XMLSEC_OPENSSL_096 */
EVP_SignUpdate(&(ctx->digestCtx), xmlSecBufferGetData(in), inSize);
#endif /* XMLSEC_OPENSSL_096 */
} else {
#ifndef XMLSEC_OPENSSL_096
ret = EVP_VerifyUpdate(&(ctx->digestCtx), xmlSecBufferGetData(in), inSize);
if(ret != 1) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"EVP_VerifyUpdate",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
#else /* XMLSEC_OPENSSL_096 */
EVP_VerifyUpdate(&(ctx->digestCtx), xmlSecBufferGetData(in), inSize);
#endif /* XMLSEC_OPENSSL_096 */
}
ret = xmlSecBufferRemoveHead(in, inSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferRemoveHead",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
}
if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) {
xmlSecAssert2(outSize == 0, -1);
if(transform->operation == xmlSecTransformOperationSign) {
unsigned int signSize;
/* this is a hack: for rsa signatures
* we get size from EVP_PKEY_size(),
* for dsa signature we use a fixed constant */
signSize = EVP_PKEY_size(ctx->pKey);
#ifndef XMLSEC_NO_DSA
if(signSize < XMLSEC_OPENSSL_DSA_SIGNATURE_SIZE) {
signSize = XMLSEC_OPENSSL_DSA_SIGNATURE_SIZE;
}
#endif /* XMLSEC_NO_DSA */
ret = xmlSecBufferSetMaxSize(out, signSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetMaxSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%u", signSize);
return(-1);
}
ret = EVP_SignFinal(&(ctx->digestCtx), xmlSecBufferGetData(out), &signSize, ctx->pKey);
if(ret != 1) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"EVP_SignFinal",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
ret = xmlSecBufferSetSize(out, signSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%u", signSize);
return(-1);
}
}
transform->status = xmlSecTransformStatusFinished;
}
if((transform->status == xmlSecTransformStatusWorking) || (transform->status == xmlSecTransformStatusFinished)) {
/* the only way we can get here is if there is no input */
xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1);
} else {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_STATUS,
"status=%d", transform->status);
return(-1);
}
return(0);
}
static int
xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) {
xmlSecOpenSSLRsaOaepCtxPtr ctx;
xmlSecSize paramsSize;
xmlSecBufferPtr in, out;
xmlSecSize inSize, outSize;
xmlSecSize keySize;
int ret;
xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1);
xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1);
xmlSecAssert2(transformCtx != NULL, -1);
ctx = xmlSecOpenSSLRsaOaepGetCtx(transform);
xmlSecAssert2(ctx != NULL, -1);
xmlSecAssert2(ctx->pKey != NULL, -1);
xmlSecAssert2(ctx->pKey->type == EVP_PKEY_RSA, -1);
xmlSecAssert2(ctx->pKey->pkey.rsa != NULL, -1);
keySize = RSA_size(ctx->pKey->pkey.rsa);
xmlSecAssert2(keySize > 0, -1);
in = &(transform->inBuf);
out = &(transform->outBuf);
inSize = xmlSecBufferGetSize(in);
outSize = xmlSecBufferGetSize(out);
xmlSecAssert2(outSize == 0, -1);
/* the encoded size is equal to the keys size so we could not
* process more than that */
if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_SIZE,
"%d when expected less than %d", inSize, keySize);
return(-1);
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_SIZE,
"%d when expected %d", inSize, keySize);
return(-1);
}
outSize = keySize;
ret = xmlSecBufferSetMaxSize(out, outSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetMaxSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", outSize);
return(-1);
}
paramsSize = xmlSecBufferGetSize(&(ctx->oaepParams));
if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize == 0)) {
/* encode w/o OAEPParams --> simple */
ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING);
if(ret <= 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"RSA_public_encrypt(RSA_PKCS1_OAEP_PADDING)",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
outSize = ret;
} else if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize > 0)) {
xmlSecAssert2(xmlSecBufferGetData(&(ctx->oaepParams)) != NULL, -1);
/* add space for padding */
ret = xmlSecBufferSetMaxSize(in, keySize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetMaxSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", keySize);
return(-1);
}
/* add padding */
ret = RSA_padding_add_PKCS1_OAEP(xmlSecBufferGetData(in), keySize,
xmlSecBufferGetData(in), inSize,
xmlSecBufferGetData(&(ctx->oaepParams)),
paramsSize);
if(ret != 1) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"RSA_padding_add_PKCS1_OAEP",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
inSize = keySize;
/* encode with OAEPParams */
ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
ctx->pKey->pkey.rsa, RSA_NO_PADDING);
if(ret <= 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"RSA_public_encrypt(RSA_NO_PADDING)",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
outSize = ret;
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize == 0)) {
ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING);
if(ret <= 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"RSA_private_decrypt(RSA_PKCS1_OAEP_PADDING)",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
outSize = ret;
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize != 0)) {
BIGNUM bn;
ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
ctx->pKey->pkey.rsa, RSA_NO_PADDING);
if(ret <= 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"RSA_private_decrypt(RSA_NO_PADDING)",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
outSize = ret;
/*
* the private decrypt w/o padding adds '0's at the begginning.
* it's not clear for me can I simply skip all '0's from the
* beggining so I have to do decode it back to BIGNUM and dump
* buffer again
*/
BN_init(&bn);
if(BN_bin2bn(xmlSecBufferGetData(out), outSize, &bn) == NULL) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"BN_bin2bn",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
"size=%d", outSize);
BN_clear_free(&bn);
return(-1);
}
ret = BN_bn2bin(&bn, xmlSecBufferGetData(out));
if(ret <= 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"BN_bn2bin",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
BN_clear_free(&bn);
return(-1);
}
BN_clear_free(&bn);
outSize = ret;
ret = RSA_padding_check_PKCS1_OAEP(xmlSecBufferGetData(out), outSize,
xmlSecBufferGetData(out), outSize,
keySize,
xmlSecBufferGetData(&(ctx->oaepParams)),
paramsSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"RSA_padding_check_PKCS1_OAEP",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
XMLSEC_ERRORS_NO_MESSAGE);
return(-1);
}
outSize = ret;
} else {
xmlSecAssert2("we could not be here" == NULL, -1);
return(-1);
}
ret = xmlSecBufferSetSize(out, outSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", outSize);
return(-1);
}
ret = xmlSecBufferRemoveHead(in, inSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferRemoveHead",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", inSize);
return(-1);
}
return(0);
}
static int
xmlSecOpenSSLRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) {
xmlSecOpenSSLRsaPkcs1CtxPtr ctx;
xmlSecBufferPtr in, out;
xmlSecSize inSize, outSize;
xmlSecSize keySize;
int ret;
xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id), -1);
xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaPkcs1Size), -1);
xmlSecAssert2(transformCtx != NULL, -1);
ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform);
xmlSecAssert2(ctx != NULL, -1);
xmlSecAssert2(ctx->pKey != NULL, -1);
xmlSecAssert2(ctx->pKey->type == EVP_PKEY_RSA, -1);
xmlSecAssert2(ctx->pKey->pkey.rsa != NULL, -1);
keySize = RSA_size(ctx->pKey->pkey.rsa);
xmlSecAssert2(keySize > 0, -1);
in = &(transform->inBuf);
out = &(transform->outBuf);
inSize = xmlSecBufferGetSize(in);
outSize = xmlSecBufferGetSize(out);
xmlSecAssert2(outSize == 0, -1);
/* the encoded size is equal to the keys size so we could not
* process more than that */
if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_SIZE,
"%d when expected less than %d", inSize, keySize);
return(-1);
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
NULL,
XMLSEC_ERRORS_R_INVALID_SIZE,
"%d when expected %d", inSize, keySize);
return(-1);
}
outSize = keySize;
ret = xmlSecBufferSetMaxSize(out, outSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetMaxSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", outSize);
return(-1);
}
if(transform->operation == xmlSecTransformOperationEncrypt) {
ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
ctx->pKey->pkey.rsa, RSA_PKCS1_PADDING);
if(ret <= 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"RSA_public_encrypt",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
"size=%d", inSize);
return(-1);
}
outSize = ret;
} else {
ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
ctx->pKey->pkey.rsa, RSA_PKCS1_PADDING);
if(ret <= 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"RSA_private_decrypt",
XMLSEC_ERRORS_R_CRYPTO_FAILED,
"size=%d", inSize);
return(-1);
}
outSize = ret;
}
ret = xmlSecBufferSetSize(out, outSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferSetSize",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", outSize);
return(-1);
}
ret = xmlSecBufferRemoveHead(in, inSize);
if(ret < 0) {
xmlSecError(XMLSEC_ERRORS_HERE,
xmlSecErrorsSafeString(xmlSecTransformGetName(transform)),
"xmlSecBufferRemoveHead",
XMLSEC_ERRORS_R_XMLSEC_FAILED,
"size=%d", inSize);
return(-1);
}
return(0);
}
static int
xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) {
xmlSecOpenSSLRsaOaepCtxPtr ctx;
xmlSecSize paramsSize;
xmlSecBufferPtr in, out;
xmlSecSize inSize, outSize;
xmlSecSize keySize;
RSA *rsa;
int ret;
xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1);
xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1);
xmlSecAssert2(transformCtx != NULL, -1);
ctx = xmlSecOpenSSLRsaOaepGetCtx(transform);
xmlSecAssert2(ctx != NULL, -1);
xmlSecAssert2(ctx->pKey != NULL, -1);
xmlSecAssert2(EVP_PKEY_base_id(ctx->pKey) == EVP_PKEY_RSA, -1);
rsa = EVP_PKEY_get0_RSA(ctx->pKey);
xmlSecAssert2(rsa != NULL, -1);
keySize = RSA_size(rsa);
xmlSecAssert2(keySize > 0, -1);
in = &(transform->inBuf);
out = &(transform->outBuf);
inSize = xmlSecBufferGetSize(in);
outSize = xmlSecBufferGetSize(out);
xmlSecAssert2(outSize == 0, -1);
/* the encoded size is equal to the keys size so we could not
* process more than that */
if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) {
xmlSecInvalidSizeLessThanError("Input data", inSize, keySize,
xmlSecTransformGetName(transform));
return(-1);
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) {
xmlSecInvalidSizeError("Input data", inSize, keySize,
xmlSecTransformGetName(transform));
return(-1);
}
outSize = keySize;
ret = xmlSecBufferSetMaxSize(out, outSize);
if(ret < 0) {
xmlSecInternalError2("xmlSecBufferSetMaxSize",
xmlSecTransformGetName(transform),
"size=%d", outSize);
return(-1);
}
paramsSize = xmlSecBufferGetSize(&(ctx->oaepParams));
if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize == 0)) {
/* encode w/o OAEPParams --> simple */
ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
rsa, RSA_PKCS1_OAEP_PADDING);
if(ret <= 0) {
xmlSecOpenSSLError("RSA_public_encrypt(RSA_PKCS1_OAEP_PADDING)",
xmlSecTransformGetName(transform));
return(-1);
}
outSize = ret;
} else if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize > 0)) {
xmlSecAssert2(xmlSecBufferGetData(&(ctx->oaepParams)) != NULL, -1);
/* add space for padding */
ret = xmlSecBufferSetMaxSize(in, keySize);
if(ret < 0) {
xmlSecInternalError2("xmlSecBufferSetMaxSize",
xmlSecTransformGetName(transform),
"size=%d", keySize);
return(-1);
}
/* add padding */
ret = RSA_padding_add_PKCS1_OAEP(xmlSecBufferGetData(in), keySize,
xmlSecBufferGetData(in), inSize,
xmlSecBufferGetData(&(ctx->oaepParams)),
paramsSize);
if(ret != 1) {
xmlSecOpenSSLError("RSA_padding_add_PKCS1_OAEP",
xmlSecTransformGetName(transform));
return(-1);
}
inSize = keySize;
/* encode with OAEPParams */
ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
rsa, RSA_NO_PADDING);
if(ret <= 0) {
xmlSecOpenSSLError("RSA_public_encrypt(RSA_NO_PADDING)",
xmlSecTransformGetName(transform));
return(-1);
}
outSize = ret;
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize == 0)) {
ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
rsa, RSA_PKCS1_OAEP_PADDING);
if(ret <= 0) {
xmlSecOpenSSLError("RSA_private_decrypt(RSA_PKCS1_OAEP_PADDING)",
xmlSecTransformGetName(transform));
return(-1);
}
outSize = ret;
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize != 0)) {
BIGNUM * bn;
bn = BN_new();
if(bn == NULL) {
xmlSecOpenSSLError("BN_new()",
xmlSecTransformGetName(transform));
return(-1);
}
ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
rsa, RSA_NO_PADDING);
if(ret <= 0) {
xmlSecOpenSSLError("RSA_private_decrypt(RSA_NO_PADDING)",
xmlSecTransformGetName(transform));
BN_free(bn);
return(-1);
}
outSize = ret;
/*
* the private decrypt w/o padding adds '0's at the begginning.
* it's not clear for me can I simply skip all '0's from the
* beggining so I have to do decode it back to BIGNUM and dump
* buffer again
*/
if(BN_bin2bn(xmlSecBufferGetData(out), outSize, bn) == NULL) {
xmlSecOpenSSLError2("BN_bin2bn",
xmlSecTransformGetName(transform),
"size=%lu", (unsigned long)outSize);
BN_free(bn);
return(-1);
}
ret = BN_bn2bin(bn, xmlSecBufferGetData(out));
if(ret <= 0) {
xmlSecOpenSSLError("BN_bn2bin",
xmlSecTransformGetName(transform));
BN_free(bn);
return(-1);
}
BN_free(bn);
outSize = ret;
ret = RSA_padding_check_PKCS1_OAEP(xmlSecBufferGetData(out), outSize,
xmlSecBufferGetData(out), outSize,
keySize,
xmlSecBufferGetData(&(ctx->oaepParams)),
paramsSize);
if(ret < 0) {
xmlSecOpenSSLError("RSA_padding_check_PKCS1_OAEP",
xmlSecTransformGetName(transform));
return(-1);
}
outSize = ret;
} else {
xmlSecOtherError3(XMLSEC_ERRORS_R_INVALID_OPERATION,
xmlSecTransformGetName(transform),
"Unexpected transform operation: %ld; paramsSize: %ld",
(long int)transform->operation, (long int)paramsSize);
return(-1);
}
ret = xmlSecBufferSetSize(out, outSize);
if(ret < 0) {
xmlSecInternalError2("xmlSecBufferSetSize",
xmlSecTransformGetName(transform),
"size=%d", outSize);
return(-1);
}
ret = xmlSecBufferRemoveHead(in, inSize);
if(ret < 0) {
xmlSecInternalError2("xmlSecBufferRemoveHead",
xmlSecTransformGetName(transform),
"size=%d", inSize);
return(-1);
}
return(0);
}
static int
xmlSecOpenSSLRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) {
xmlSecOpenSSLRsaPkcs1CtxPtr ctx;
xmlSecBufferPtr in, out;
xmlSecSize inSize, outSize;
xmlSecSize keySize;
RSA *rsa;
int ret;
xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id), -1);
xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1);
xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaPkcs1Size), -1);
xmlSecAssert2(transformCtx != NULL, -1);
ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform);
xmlSecAssert2(ctx != NULL, -1);
xmlSecAssert2(ctx->pKey != NULL, -1);
xmlSecAssert2(EVP_PKEY_base_id(ctx->pKey) == EVP_PKEY_RSA, -1);
rsa = EVP_PKEY_get0_RSA(ctx->pKey);
xmlSecAssert2(rsa != NULL, -1);
keySize = RSA_size(rsa);
xmlSecAssert2(keySize > 0, -1);
in = &(transform->inBuf);
out = &(transform->outBuf);
inSize = xmlSecBufferGetSize(in);
outSize = xmlSecBufferGetSize(out);
xmlSecAssert2(outSize == 0, -1);
/* the encoded size is equal to the keys size so we could not
* process more than that */
if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) {
xmlSecInvalidSizeLessThanError("Input data", inSize, keySize,
xmlSecTransformGetName(transform));
return(-1);
} else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) {
xmlSecInvalidSizeError("Input data", inSize, keySize,
xmlSecTransformGetName(transform));
return(-1);
}
outSize = keySize;
ret = xmlSecBufferSetMaxSize(out, outSize);
if(ret < 0) {
xmlSecInternalError2("xmlSecBufferSetMaxSize",
xmlSecTransformGetName(transform),
"size=%d", outSize);
return(-1);
}
if(transform->operation == xmlSecTransformOperationEncrypt) {
ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
rsa, RSA_PKCS1_PADDING);
if(ret <= 0) {
xmlSecOpenSSLError2("RSA_public_encrypt",
xmlSecTransformGetName(transform),
"size=%lu", (unsigned long)inSize);
return(-1);
}
outSize = ret;
} else {
ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in),
xmlSecBufferGetData(out),
rsa, RSA_PKCS1_PADDING);
if(ret <= 0) {
xmlSecOpenSSLError2("RSA_private_decrypt",
xmlSecTransformGetName(transform),
"size=%lu", (unsigned long)inSize);
return(-1);
}
outSize = ret;
}
ret = xmlSecBufferSetSize(out, outSize);
if(ret < 0) {
xmlSecInternalError2("xmlSecBufferSetSize",
xmlSecTransformGetName(transform),
"size=%d", outSize);
return(-1);
}
ret = xmlSecBufferRemoveHead(in, inSize);
if(ret < 0) {
xmlSecInternalError2("xmlSecBufferRemoveHead",
xmlSecTransformGetName(transform),
"size=%d", inSize);
return(-1);
}
return(0);
}
