PyObject *xmlsec_BufferSetMaxSize(PyObject *self, PyObject *args) { PyObject *buf_obj; xmlSecBufferPtr buf; xmlSecSize size; if (CheckArgs(args, "OI:bufferSetMaxSize")) { if (!PyArg_ParseTuple(args, "Oi:bufferSetMaxSize", &buf_obj, &size)) return NULL; } else return NULL; buf = xmlSecBufferPtr_get(buf_obj); return (wrap_int(xmlSecBufferSetMaxSize(buf, size))); }
static int xmlSecMSCryptoRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) { xmlSecMSCryptoRsaPkcs1CtxPtr ctx; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize; xmlSecSize keySize; int ret; HCRYPTKEY hKey = 0; DWORD dwInLen; DWORD dwBufLen; DWORD dwOutLen; BYTE * outBuf; BYTE * inBuf; int i; xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecMSCryptoTransformRsaPkcs1Id), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecMSCryptoRsaPkcs1Size), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecMSCryptoRsaPkcs1GetCtx(transform); xmlSecAssert2(ctx != NULL, -1); xmlSecAssert2(ctx->data != NULL, -1); keySize = xmlSecKeyDataGetSize(ctx->data) / 8; xmlSecAssert2(keySize > 0, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); /* the encoded size is equal to the keys size so we could not * process more than that */ if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "%d when expected less than %d", inSize, keySize); return(-1); } else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "%d when expected %d", inSize, keySize); return(-1); } outSize = keySize; ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { BYTE ch; if(inSize > outSize) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "inSize=%d;outSize=%d", inSize, outSize); return(-1); } ret = xmlSecBufferSetData(out, xmlSecBufferGetData(in), inSize); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetData", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", inSize); return(-1); } dwInLen = inSize; dwBufLen = outSize; if (0 == (hKey = xmlSecMSCryptoKeyDataGetKey(ctx->data, xmlSecKeyDataTypePublic))) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, NULL, "xmlSecMSCryptoKeyDataGetKey", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return (-1); } outBuf = xmlSecBufferGetData(out); xmlSecAssert2(outBuf != NULL, -1); if (!CryptEncrypt(hKey, 0, TRUE, 0, outBuf, &dwInLen, dwBufLen)) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, NULL, "CryptEncrypt", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return (-1); } /* The output of CryptEncrypt is in little-endian format, so we have to convert to * big-endian first. */ for(i = 0; i < outSize / 2; i++) { ch = outBuf[i]; outBuf[i] = outBuf[outSize - (i + 1)]; outBuf[outSize - (i + 1)] = ch; } } else { dwOutLen = inSize; /* The input of CryptDecrypt is expected to be little-endian, * so we have to convert from big-endian to little endian. */ inBuf = xmlSecBufferGetData(in); outBuf = xmlSecBufferGetData(out); xmlSecAssert2(inBuf != 0, -1); xmlSecAssert2(outBuf != 0, -1); for (i = 0; i < inSize; i++) { outBuf[i] = inBuf[inSize - (i + 1)]; } if (0 == (hKey = xmlSecMSCryptoKeyDataGetDecryptKey(ctx->data))) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, NULL, "xmlSecMSCryptoKeyDataGetKey", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return (-1); } if (!CryptDecrypt(hKey, 0, TRUE, 0, outBuf, &dwOutLen)) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "CryptDecrypt", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = dwOutLen; } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", inSize); return(-1); } return(0); }
static int xmlSecMSCryptoKWDes3Execute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) { xmlSecMSCryptoKWDes3CtxPtr ctx; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize, keySize; int ret; xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecMSCryptoTransformKWDes3Id), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecMSCryptoKWDes3Size), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecMSCryptoKWDes3GetCtx(transform); xmlSecAssert2(ctx != NULL, -1); keySize = xmlSecBufferGetSize(&(ctx->keyBuffer)); xmlSecAssert2(keySize == XMLSEC_KW_DES3_KEY_LENGTH, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); if(transform->status == xmlSecTransformStatusNone) { transform->status = xmlSecTransformStatusWorking; } if((transform->status == xmlSecTransformStatusWorking) && (last == 0)) { /* just do nothing */ } else if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) { if((inSize % XMLSEC_KW_DES3_BLOCK_LENGTH) != 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "%d bytes - not %d bytes aligned", inSize, XMLSEC_KW_DES3_BLOCK_LENGTH); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { /* the encoded key might be 16 bytes longer plus one block just in case */ outSize = inSize + XMLSEC_KW_DES3_IV_LENGTH + XMLSEC_KW_DES3_BLOCK_LENGTH + XMLSEC_KW_DES3_BLOCK_LENGTH; } else { /* just in case, add a block */ outSize = inSize + XMLSEC_KW_DES3_BLOCK_LENGTH; } ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { ret = xmlSecKWDes3Encode(&xmlSecMSCryptoKWDes3ImplKlass, ctx, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(out), outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecKWDes3Encode", XMLSEC_ERRORS_R_XMLSEC_FAILED, "key=%d,in=%d,out=%d", keySize, inSize, outSize); return(-1); } outSize = ret; } else { ret = xmlSecKWDes3Decode(&xmlSecMSCryptoKWDes3ImplKlass, ctx, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(out), outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecKWDes3Decode", XMLSEC_ERRORS_R_XMLSEC_FAILED, "key=%d,in=%d,out=%d", keySize, inSize, outSize); return(-1); } outSize = ret; } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", inSize); return(-1); } transform->status = xmlSecTransformStatusFinished; } else if(transform->status == xmlSecTransformStatusFinished) { /* the only way we can get here is if there is no input */ xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1); } else { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_STATUS, "status=%d", transform->status); return(-1); } return(0); }
static int xmlSecBase64Execute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) { xmlSecBase64CtxPtr ctx; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize, outLen; int ret; xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecTransformBase64Id), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncode) || (transform->operation == xmlSecTransformOperationDecode), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecBase64GetCtx(transform); xmlSecAssert2(ctx != NULL, -1); in = &(transform->inBuf); out = &(transform->outBuf); if(transform->status == xmlSecTransformStatusNone) { ctx->encode = (transform->operation == xmlSecTransformOperationEncode) ? 1 : 0; transform->status = xmlSecTransformStatusWorking; } switch(transform->status) { case xmlSecTransformStatusWorking: inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); if(inSize > 0) { if(ctx->encode != 0) { outLen = 4 * inSize / 3 + 8; if(ctx->columns > 0) { outLen += inSize / ctx->columns + 4; } } else { outLen = 3 * inSize / 4 + 8; } ret = xmlSecBufferSetMaxSize(out, outSize + outLen); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize + outLen); return(-1); } /* encode/decode the next chunk */ ret = xmlSecBase64CtxUpdate(ctx, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(out) + outSize, outLen); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBase64CtxUpdate", XMLSEC_ERRORS_R_XMLSEC_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outLen = ret; /* set correct size */ ret = xmlSecBufferSetSize(out, outSize + outLen); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize + outLen); return(-1); } /* remove chunk from input */ ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", inSize); return(-1); } } if(last) { outSize = xmlSecBufferGetSize(out); ret = xmlSecBufferSetMaxSize(out, outSize + 16); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize + 16); return(-1); } /* add from ctx buffer */ ret = xmlSecBase64CtxFinal(ctx, xmlSecBufferGetData(out) + outSize, 16); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBase64CtxFinal", XMLSEC_ERRORS_R_XMLSEC_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outLen = ret; /* set correct size */ ret = xmlSecBufferSetSize(out, outSize + outLen); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize + outLen); return(-1); } transform->status = xmlSecTransformStatusFinished; } break; case xmlSecTransformStatusFinished: /* the only way we can get here is if there is no input */ xmlSecAssert2(xmlSecBufferGetSize(in) == 0, -1); break; default: xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_STATUS, "status=%d", transform->status); return(-1); } return(0); }
static int xmlSecNssKWAesExecute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) { xmlSecBufferPtr in, out, key; xmlSecSize inSize, outSize, keySize, expectedKeySize; int ret; xmlSecAssert2(xmlSecNssKWAesCheckId(transform), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecNssKWAesSize), -1); xmlSecAssert2(transformCtx != NULL, -1); key = xmlSecNssKWAesGetKey(transform); xmlSecAssert2(key != NULL, -1); keySize = xmlSecBufferGetSize(key); expectedKeySize = xmlSecNssKWAesGetKeySize(transform); xmlSecAssert2(keySize == expectedKeySize, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); if(transform->status == xmlSecTransformStatusNone) { transform->status = xmlSecTransformStatusWorking; } if((transform->status == xmlSecTransformStatusWorking) && (last == 0)) { /* just do nothing */ } else if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) { if((inSize % 8) != 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "size=%d(not 8 bytes aligned)", inSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { /* the encoded key might be 8 bytes longer plus 8 bytes just in case */ outSize = inSize + XMLSEC_NSS_KW_AES_MAGIC_BLOCK_SIZE + XMLSEC_NSS_AES_BLOCK_SIZE; } else { outSize = inSize + XMLSEC_NSS_AES_BLOCK_SIZE; } ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "outSize=%d", outSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { ret = xmlSecNssKWAesOp(xmlSecBufferGetData(key), keySize, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(out), outSize, 1); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecNssKWAesOp", XMLSEC_ERRORS_R_XMLSEC_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = ret; } else { ret = xmlSecNssKWAesOp(xmlSecBufferGetData(key), keySize, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(out), outSize, 0); if(ret < 0) { xmlSecErr_a_ignorar5(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecNssKWAesOp", XMLSEC_ERRORS_R_XMLSEC_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = ret; } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, g "outSize=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, "inSize%d", inSize); return(-1); } transform->status = xmlSecTransformStatusFinished; } else if(transform->status == xmlSecTransformStatusFinished) { /* the only way we can get here is if there is no input */ xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1); } else { xmlSecErr_a_ignorar6(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_STATUS, "status=%d", transform->status); return(-1); } return(0); }
/** * xmlSecBnFromString: * @bn: the pointer to BN. * @str: the string with BN. * @base: the base for @str. * * Reads @bn from string @str assuming it has base @base. * * Returns 0 on success or a negative value if an error occurs. */ EXPORT_C int xmlSecBnFromString(xmlSecBnPtr bn, const xmlChar* str, xmlSecSize base) { xmlSecSize i, len, size; xmlSecByte ch; xmlSecByte* data; int positive; int nn; int ret; xmlSecAssert2(bn != NULL, -1); xmlSecAssert2(str != NULL, -1); xmlSecAssert2(base > 1, -1); xmlSecAssert2(base <= sizeof(xmlSecBnRevLookupTable), -1); /* trivial case */ len = xmlStrlen(str); if(len == 0) { return(0); } /* The result size could not exceed the input string length * because each char fits inside a byte in all cases :) * In truth, it would be likely less than 1/2 input string length * because each byte is represented by 2 chars. If needed, * buffer size would be increased by Mul/Add functions. * Finally, we can add one byte for 00 or 10 prefix. */ ret = xmlSecBufferSetMaxSize(bn, xmlSecBufferGetSize(bn) + len / 2 + 1 + 1); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, NULL, "xmlSecBnRevLookupTable", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", len / 2 + 1); return (-1); } /* figure out if it is positive or negative number */ positive = 1; i = 0; while(i < len) { ch = str[i++]; /* skip spaces */ if(isspace(ch)) { continue; } /* check if it is + or - */ if(ch == '+') { positive = 1; break; } else if(ch == '-') { positive = 0; break; } /* otherwise, it must be start of the number */ nn = xmlSecBnLookupTable[ch]; if((nn >= 0) && ((xmlSecSize)nn < base)) { xmlSecAssert2(i > 0, -1); /* no sign, positive by default */ positive = 1; --i; /* make sure that we will look at this character in next loop */ break; } else { xmlSecError(XMLSEC_ERRORS_HERE, NULL, NULL, XMLSEC_ERRORS_R_INVALID_DATA, "char=%c;base=%d", ch, base); return (-1); } } /* now parse the number itself */ while(i < len) { ch = str[i++]; if(isspace(ch)) { continue; } xmlSecAssert2(ch <(sizeof(xmlSecBnLookupTable)/sizeof(xmlSecBnLookupTable[0])), -1); nn = xmlSecBnLookupTable[ch]; if((nn < 0) || ((xmlSecSize)nn > base)) { xmlSecError(XMLSEC_ERRORS_HERE, NULL, NULL, XMLSEC_ERRORS_R_INVALID_DATA, "char=%c;base=%d", ch, base); return (-1); } ret = xmlSecBnMul(bn, base); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, NULL, "xmlSecBnMul", XMLSEC_ERRORS_R_XMLSEC_FAILED, "base=%d", base); return (-1); } ret = xmlSecBnAdd(bn, nn); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, NULL, "xmlSecBnAdd", XMLSEC_ERRORS_R_XMLSEC_FAILED, "base=%d", base); return (-1); } } /* check if we need to add 00 prefix, do this for empty bn too */ data = xmlSecBufferGetData(bn); size = xmlSecBufferGetSize(bn); if(((size > 0) && (data[0] > 127)) || (size == 0)) { ch = 0; ret = xmlSecBufferPrepend(bn, &ch, 1); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, NULL, "xmlSecBufferPrepend", XMLSEC_ERRORS_R_XMLSEC_FAILED, "base=%d", base); return (-1); } } /* do 2's compliment and add 1 to represent negative value */ if(positive == 0) { data = xmlSecBufferGetData(bn); size = xmlSecBufferGetSize(bn); for(i = 0; i < size; ++i) { data[i] ^= 0xFF; } ret = xmlSecBnAdd(bn, 1); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, NULL, "xmlSecBnAdd", XMLSEC_ERRORS_R_XMLSEC_FAILED, "base=%d", base); return (-1); } } return(0); }
static int xmlSecNssKWAesExecute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) { xmlSecNssKWAesCtxPtr ctx; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize, keySize; int ret; xmlSecAssert2(xmlSecNssKWAesCheckId(transform), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecNssKWAesSize), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecNssKWAesGetCtx(transform); xmlSecAssert2(ctx != NULL, -1); keySize = xmlSecBufferGetSize(&(ctx->keyBuffer)); xmlSecAssert2(keySize == ctx->keyExpectedSize, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); if(transform->status == xmlSecTransformStatusNone) { transform->status = xmlSecTransformStatusWorking; } if((transform->status == xmlSecTransformStatusWorking) && (last == 0)) { /* just do nothing */ } else if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) { if((inSize % 8) != 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "size=%d(not 8 bytes aligned)", inSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { /* the encoded key might be 8 bytes longer plus 8 bytes just in case */ outSize = inSize + XMLSEC_KW_AES_MAGIC_BLOCK_SIZE + XMLSEC_KW_AES_BLOCK_SIZE; } else { outSize = inSize + XMLSEC_KW_AES_BLOCK_SIZE; } ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "outSize=%d", outSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { PK11SymKey *aeskey = NULL; /* create key */ aeskey = xmlSecNssMakeAesKey(xmlSecBufferGetData(&(ctx->keyBuffer)), keySize, 1); /* encrypt */ if(aeskey == NULL) { xmlSecError(XMLSEC_ERRORS_HERE, NULL, "xmlSecNssMakeAesKey", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } /* encrypt */ ret = xmlSecKWAesEncode(&xmlSecNssKWAesKlass, aeskey, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(out), outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecKWAesEncode", XMLSEC_ERRORS_R_XMLSEC_FAILED, XMLSEC_ERRORS_NO_MESSAGE); PK11_FreeSymKey(aeskey); return(-1); } outSize = ret; PK11_FreeSymKey(aeskey); } else { PK11SymKey *aeskey = NULL; /* create key */ aeskey = xmlSecNssMakeAesKey(xmlSecBufferGetData(&(ctx->keyBuffer)), keySize, 0); /* decrypt */ if(aeskey == NULL) { xmlSecError(XMLSEC_ERRORS_HERE, NULL, "xmlSecNssMakeAesKey", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } /* decrypt */ ret = xmlSecKWAesDecode(&xmlSecNssKWAesKlass, aeskey, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(out), outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecKWAesDecode", XMLSEC_ERRORS_R_XMLSEC_FAILED, XMLSEC_ERRORS_NO_MESSAGE); PK11_FreeSymKey(aeskey); return(-1); } outSize = ret; PK11_FreeSymKey(aeskey); } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "outSize=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, "inSize%d", inSize); return(-1); } transform->status = xmlSecTransformStatusFinished; } else if(transform->status == xmlSecTransformStatusFinished) { /* the only way we can get here is if there is no input */ xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1); } else { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_STATUS, "status=%d", transform->status); return(-1); } return(0); }
static int xmlSecOpenSSLEvpSignatureExecute(xmlSecTransformPtr transform, int last, xmlSecTransformCtxPtr transformCtx) { xmlSecOpenSSLEvpSignatureCtxPtr ctx; xmlSecBufferPtr in, out; xmlSecSize inSize; xmlSecSize outSize; int ret; xmlSecAssert2(xmlSecOpenSSLEvpSignatureCheckId(transform), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationSign) || (transform->operation == xmlSecTransformOperationVerify), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLEvpSignatureSize), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecOpenSSLEvpSignatureGetCtx(transform); xmlSecAssert2(ctx != NULL, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); ctx = xmlSecOpenSSLEvpSignatureGetCtx(transform); xmlSecAssert2(ctx != NULL, -1); xmlSecAssert2(ctx->digest != NULL, -1); xmlSecAssert2(ctx->pKey != NULL, -1); if(transform->status == xmlSecTransformStatusNone) { xmlSecAssert2(outSize == 0, -1); if(transform->operation == xmlSecTransformOperationSign) { #ifndef XMLSEC_OPENSSL_096 ret = EVP_SignInit(&(ctx->digestCtx), ctx->digest); if(ret != 1) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "EVP_SignInit", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } #else /* XMLSEC_OPENSSL_096 */ EVP_SignInit(&(ctx->digestCtx), ctx->digest); #endif /* XMLSEC_OPENSSL_096 */ } else { #ifndef XMLSEC_OPENSSL_096 ret = EVP_VerifyInit(&(ctx->digestCtx), ctx->digest); if(ret != 1) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "EVP_VerifyInit", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } #else /* XMLSEC_OPENSSL_096 */ EVP_VerifyInit(&(ctx->digestCtx), ctx->digest); #endif /* XMLSEC_OPENSSL_096 */ } transform->status = xmlSecTransformStatusWorking; } if((transform->status == xmlSecTransformStatusWorking) && (inSize > 0)) { xmlSecAssert2(outSize == 0, -1); if(transform->operation == xmlSecTransformOperationSign) { #ifndef XMLSEC_OPENSSL_096 ret = EVP_SignUpdate(&(ctx->digestCtx), xmlSecBufferGetData(in), inSize); if(ret != 1) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "EVP_SignUpdate", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } #else /* XMLSEC_OPENSSL_096 */ EVP_SignUpdate(&(ctx->digestCtx), xmlSecBufferGetData(in), inSize); #endif /* XMLSEC_OPENSSL_096 */ } else { #ifndef XMLSEC_OPENSSL_096 ret = EVP_VerifyUpdate(&(ctx->digestCtx), xmlSecBufferGetData(in), inSize); if(ret != 1) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "EVP_VerifyUpdate", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } #else /* XMLSEC_OPENSSL_096 */ EVP_VerifyUpdate(&(ctx->digestCtx), xmlSecBufferGetData(in), inSize); #endif /* XMLSEC_OPENSSL_096 */ } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } } if((transform->status == xmlSecTransformStatusWorking) && (last != 0)) { xmlSecAssert2(outSize == 0, -1); if(transform->operation == xmlSecTransformOperationSign) { unsigned int signSize; /* this is a hack: for rsa signatures * we get size from EVP_PKEY_size(), * for dsa signature we use a fixed constant */ signSize = EVP_PKEY_size(ctx->pKey); #ifndef XMLSEC_NO_DSA if(signSize < XMLSEC_OPENSSL_DSA_SIGNATURE_SIZE) { signSize = XMLSEC_OPENSSL_DSA_SIGNATURE_SIZE; } #endif /* XMLSEC_NO_DSA */ ret = xmlSecBufferSetMaxSize(out, signSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%u", signSize); return(-1); } ret = EVP_SignFinal(&(ctx->digestCtx), xmlSecBufferGetData(out), &signSize, ctx->pKey); if(ret != 1) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "EVP_SignFinal", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } ret = xmlSecBufferSetSize(out, signSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%u", signSize); return(-1); } } transform->status = xmlSecTransformStatusFinished; } if((transform->status == xmlSecTransformStatusWorking) || (transform->status == xmlSecTransformStatusFinished)) { /* the only way we can get here is if there is no input */ xmlSecAssert2(xmlSecBufferGetSize(&(transform->inBuf)) == 0, -1); } else { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_STATUS, "status=%d", transform->status); return(-1); } return(0); }
static int xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) { xmlSecOpenSSLRsaOaepCtxPtr ctx; xmlSecSize paramsSize; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize; xmlSecSize keySize; int ret; xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecOpenSSLRsaOaepGetCtx(transform); xmlSecAssert2(ctx != NULL, -1); xmlSecAssert2(ctx->pKey != NULL, -1); xmlSecAssert2(ctx->pKey->type == EVP_PKEY_RSA, -1); xmlSecAssert2(ctx->pKey->pkey.rsa != NULL, -1); keySize = RSA_size(ctx->pKey->pkey.rsa); xmlSecAssert2(keySize > 0, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); /* the encoded size is equal to the keys size so we could not * process more than that */ if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "%d when expected less than %d", inSize, keySize); return(-1); } else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "%d when expected %d", inSize, keySize); return(-1); } outSize = keySize; ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } paramsSize = xmlSecBufferGetSize(&(ctx->oaepParams)); if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize == 0)) { /* encode w/o OAEPParams --> simple */ ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING); if(ret <= 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_public_encrypt(RSA_PKCS1_OAEP_PADDING)", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = ret; } else if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize > 0)) { xmlSecAssert2(xmlSecBufferGetData(&(ctx->oaepParams)) != NULL, -1); /* add space for padding */ ret = xmlSecBufferSetMaxSize(in, keySize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", keySize); return(-1); } /* add padding */ ret = RSA_padding_add_PKCS1_OAEP(xmlSecBufferGetData(in), keySize, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(&(ctx->oaepParams)), paramsSize); if(ret != 1) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_padding_add_PKCS1_OAEP", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } inSize = keySize; /* encode with OAEPParams */ ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), ctx->pKey->pkey.rsa, RSA_NO_PADDING); if(ret <= 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_public_encrypt(RSA_NO_PADDING)", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = ret; } else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize == 0)) { ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), ctx->pKey->pkey.rsa, RSA_PKCS1_OAEP_PADDING); if(ret <= 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_private_decrypt(RSA_PKCS1_OAEP_PADDING)", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = ret; } else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize != 0)) { BIGNUM bn; ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), ctx->pKey->pkey.rsa, RSA_NO_PADDING); if(ret <= 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_private_decrypt(RSA_NO_PADDING)", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = ret; /* * the private decrypt w/o padding adds '0's at the begginning. * it's not clear for me can I simply skip all '0's from the * beggining so I have to do decode it back to BIGNUM and dump * buffer again */ BN_init(&bn); if(BN_bin2bn(xmlSecBufferGetData(out), outSize, &bn) == NULL) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "BN_bin2bn", XMLSEC_ERRORS_R_CRYPTO_FAILED, "size=%d", outSize); BN_clear_free(&bn); return(-1); } ret = BN_bn2bin(&bn, xmlSecBufferGetData(out)); if(ret <= 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "BN_bn2bin", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); BN_clear_free(&bn); return(-1); } BN_clear_free(&bn); outSize = ret; ret = RSA_padding_check_PKCS1_OAEP(xmlSecBufferGetData(out), outSize, xmlSecBufferGetData(out), outSize, keySize, xmlSecBufferGetData(&(ctx->oaepParams)), paramsSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_padding_check_PKCS1_OAEP", XMLSEC_ERRORS_R_CRYPTO_FAILED, XMLSEC_ERRORS_NO_MESSAGE); return(-1); } outSize = ret; } else { xmlSecAssert2("we could not be here" == NULL, -1); return(-1); } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", inSize); return(-1); } return(0); }
static int xmlSecOpenSSLRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) { xmlSecOpenSSLRsaPkcs1CtxPtr ctx; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize; xmlSecSize keySize; int ret; xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaPkcs1Size), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform); xmlSecAssert2(ctx != NULL, -1); xmlSecAssert2(ctx->pKey != NULL, -1); xmlSecAssert2(ctx->pKey->type == EVP_PKEY_RSA, -1); xmlSecAssert2(ctx->pKey->pkey.rsa != NULL, -1); keySize = RSA_size(ctx->pKey->pkey.rsa); xmlSecAssert2(keySize > 0, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); /* the encoded size is equal to the keys size so we could not * process more than that */ if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "%d when expected less than %d", inSize, keySize); return(-1); } else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), NULL, XMLSEC_ERRORS_R_INVALID_SIZE, "%d when expected %d", inSize, keySize); return(-1); } outSize = keySize; ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetMaxSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), ctx->pKey->pkey.rsa, RSA_PKCS1_PADDING); if(ret <= 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_public_encrypt", XMLSEC_ERRORS_R_CRYPTO_FAILED, "size=%d", inSize); return(-1); } outSize = ret; } else { ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), ctx->pKey->pkey.rsa, RSA_PKCS1_PADDING); if(ret <= 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "RSA_private_decrypt", XMLSEC_ERRORS_R_CRYPTO_FAILED, "size=%d", inSize); return(-1); } outSize = ret; } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferSetSize", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecError(XMLSEC_ERRORS_HERE, xmlSecErrorsSafeString(xmlSecTransformGetName(transform)), "xmlSecBufferRemoveHead", XMLSEC_ERRORS_R_XMLSEC_FAILED, "size=%d", inSize); return(-1); } return(0); }
static int xmlSecOpenSSLRsaOaepProcess(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) { xmlSecOpenSSLRsaOaepCtxPtr ctx; xmlSecSize paramsSize; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize; xmlSecSize keySize; RSA *rsa; int ret; xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaOaepId), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaOaepSize), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecOpenSSLRsaOaepGetCtx(transform); xmlSecAssert2(ctx != NULL, -1); xmlSecAssert2(ctx->pKey != NULL, -1); xmlSecAssert2(EVP_PKEY_base_id(ctx->pKey) == EVP_PKEY_RSA, -1); rsa = EVP_PKEY_get0_RSA(ctx->pKey); xmlSecAssert2(rsa != NULL, -1); keySize = RSA_size(rsa); xmlSecAssert2(keySize > 0, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); /* the encoded size is equal to the keys size so we could not * process more than that */ if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) { xmlSecInvalidSizeLessThanError("Input data", inSize, keySize, xmlSecTransformGetName(transform)); return(-1); } else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) { xmlSecInvalidSizeError("Input data", inSize, keySize, xmlSecTransformGetName(transform)); return(-1); } outSize = keySize; ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecInternalError2("xmlSecBufferSetMaxSize", xmlSecTransformGetName(transform), "size=%d", outSize); return(-1); } paramsSize = xmlSecBufferGetSize(&(ctx->oaepParams)); if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize == 0)) { /* encode w/o OAEPParams --> simple */ ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), rsa, RSA_PKCS1_OAEP_PADDING); if(ret <= 0) { xmlSecOpenSSLError("RSA_public_encrypt(RSA_PKCS1_OAEP_PADDING)", xmlSecTransformGetName(transform)); return(-1); } outSize = ret; } else if((transform->operation == xmlSecTransformOperationEncrypt) && (paramsSize > 0)) { xmlSecAssert2(xmlSecBufferGetData(&(ctx->oaepParams)) != NULL, -1); /* add space for padding */ ret = xmlSecBufferSetMaxSize(in, keySize); if(ret < 0) { xmlSecInternalError2("xmlSecBufferSetMaxSize", xmlSecTransformGetName(transform), "size=%d", keySize); return(-1); } /* add padding */ ret = RSA_padding_add_PKCS1_OAEP(xmlSecBufferGetData(in), keySize, xmlSecBufferGetData(in), inSize, xmlSecBufferGetData(&(ctx->oaepParams)), paramsSize); if(ret != 1) { xmlSecOpenSSLError("RSA_padding_add_PKCS1_OAEP", xmlSecTransformGetName(transform)); return(-1); } inSize = keySize; /* encode with OAEPParams */ ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), rsa, RSA_NO_PADDING); if(ret <= 0) { xmlSecOpenSSLError("RSA_public_encrypt(RSA_NO_PADDING)", xmlSecTransformGetName(transform)); return(-1); } outSize = ret; } else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize == 0)) { ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), rsa, RSA_PKCS1_OAEP_PADDING); if(ret <= 0) { xmlSecOpenSSLError("RSA_private_decrypt(RSA_PKCS1_OAEP_PADDING)", xmlSecTransformGetName(transform)); return(-1); } outSize = ret; } else if((transform->operation == xmlSecTransformOperationDecrypt) && (paramsSize != 0)) { BIGNUM * bn; bn = BN_new(); if(bn == NULL) { xmlSecOpenSSLError("BN_new()", xmlSecTransformGetName(transform)); return(-1); } ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), rsa, RSA_NO_PADDING); if(ret <= 0) { xmlSecOpenSSLError("RSA_private_decrypt(RSA_NO_PADDING)", xmlSecTransformGetName(transform)); BN_free(bn); return(-1); } outSize = ret; /* * the private decrypt w/o padding adds '0's at the begginning. * it's not clear for me can I simply skip all '0's from the * beggining so I have to do decode it back to BIGNUM and dump * buffer again */ if(BN_bin2bn(xmlSecBufferGetData(out), outSize, bn) == NULL) { xmlSecOpenSSLError2("BN_bin2bn", xmlSecTransformGetName(transform), "size=%lu", (unsigned long)outSize); BN_free(bn); return(-1); } ret = BN_bn2bin(bn, xmlSecBufferGetData(out)); if(ret <= 0) { xmlSecOpenSSLError("BN_bn2bin", xmlSecTransformGetName(transform)); BN_free(bn); return(-1); } BN_free(bn); outSize = ret; ret = RSA_padding_check_PKCS1_OAEP(xmlSecBufferGetData(out), outSize, xmlSecBufferGetData(out), outSize, keySize, xmlSecBufferGetData(&(ctx->oaepParams)), paramsSize); if(ret < 0) { xmlSecOpenSSLError("RSA_padding_check_PKCS1_OAEP", xmlSecTransformGetName(transform)); return(-1); } outSize = ret; } else { xmlSecOtherError3(XMLSEC_ERRORS_R_INVALID_OPERATION, xmlSecTransformGetName(transform), "Unexpected transform operation: %ld; paramsSize: %ld", (long int)transform->operation, (long int)paramsSize); return(-1); } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecInternalError2("xmlSecBufferSetSize", xmlSecTransformGetName(transform), "size=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecInternalError2("xmlSecBufferRemoveHead", xmlSecTransformGetName(transform), "size=%d", inSize); return(-1); } return(0); }
static int xmlSecOpenSSLRsaPkcs1Process(xmlSecTransformPtr transform, xmlSecTransformCtxPtr transformCtx) { xmlSecOpenSSLRsaPkcs1CtxPtr ctx; xmlSecBufferPtr in, out; xmlSecSize inSize, outSize; xmlSecSize keySize; RSA *rsa; int ret; xmlSecAssert2(xmlSecTransformCheckId(transform, xmlSecOpenSSLTransformRsaPkcs1Id), -1); xmlSecAssert2((transform->operation == xmlSecTransformOperationEncrypt) || (transform->operation == xmlSecTransformOperationDecrypt), -1); xmlSecAssert2(xmlSecTransformCheckSize(transform, xmlSecOpenSSLRsaPkcs1Size), -1); xmlSecAssert2(transformCtx != NULL, -1); ctx = xmlSecOpenSSLRsaPkcs1GetCtx(transform); xmlSecAssert2(ctx != NULL, -1); xmlSecAssert2(ctx->pKey != NULL, -1); xmlSecAssert2(EVP_PKEY_base_id(ctx->pKey) == EVP_PKEY_RSA, -1); rsa = EVP_PKEY_get0_RSA(ctx->pKey); xmlSecAssert2(rsa != NULL, -1); keySize = RSA_size(rsa); xmlSecAssert2(keySize > 0, -1); in = &(transform->inBuf); out = &(transform->outBuf); inSize = xmlSecBufferGetSize(in); outSize = xmlSecBufferGetSize(out); xmlSecAssert2(outSize == 0, -1); /* the encoded size is equal to the keys size so we could not * process more than that */ if((transform->operation == xmlSecTransformOperationEncrypt) && (inSize >= keySize)) { xmlSecInvalidSizeLessThanError("Input data", inSize, keySize, xmlSecTransformGetName(transform)); return(-1); } else if((transform->operation == xmlSecTransformOperationDecrypt) && (inSize != keySize)) { xmlSecInvalidSizeError("Input data", inSize, keySize, xmlSecTransformGetName(transform)); return(-1); } outSize = keySize; ret = xmlSecBufferSetMaxSize(out, outSize); if(ret < 0) { xmlSecInternalError2("xmlSecBufferSetMaxSize", xmlSecTransformGetName(transform), "size=%d", outSize); return(-1); } if(transform->operation == xmlSecTransformOperationEncrypt) { ret = RSA_public_encrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), rsa, RSA_PKCS1_PADDING); if(ret <= 0) { xmlSecOpenSSLError2("RSA_public_encrypt", xmlSecTransformGetName(transform), "size=%lu", (unsigned long)inSize); return(-1); } outSize = ret; } else { ret = RSA_private_decrypt(inSize, xmlSecBufferGetData(in), xmlSecBufferGetData(out), rsa, RSA_PKCS1_PADDING); if(ret <= 0) { xmlSecOpenSSLError2("RSA_private_decrypt", xmlSecTransformGetName(transform), "size=%lu", (unsigned long)inSize); return(-1); } outSize = ret; } ret = xmlSecBufferSetSize(out, outSize); if(ret < 0) { xmlSecInternalError2("xmlSecBufferSetSize", xmlSecTransformGetName(transform), "size=%d", outSize); return(-1); } ret = xmlSecBufferRemoveHead(in, inSize); if(ret < 0) { xmlSecInternalError2("xmlSecBufferRemoveHead", xmlSecTransformGetName(transform), "size=%d", inSize); return(-1); } return(0); }