From e369684b991f3fcf8333389ddb045f1c05b3655e Mon Sep 17 00:00:00 2001 From: Stefan Berger Date: Fri, 13 Jun 2025 13:42:55 -0400 Subject: [PATCH] tpm2: Helpers,OpenSSL: Convert some TPM_RC_FAILURE to TPM_RC_MEMORY Convert those TPM_RC_FAILURE return codes to TPM_RC_MEMORY where it is certain that it is only an issue related to no memory being available rather than anything else. The following OpenSSL functions qualify for this: - BN_CTX_new & BN_CTX_new_ex - BN_new The following ones do not qualify: - RSA_new: could have failure with ENGINE_init() and ENGINE_get_RSA() - EC_KEY_new: could have failure with ENGINE_init() and ENGINE_get_EC() - BigInitialized: initializer could be NULL - BN_bin2bn: length could be 0 Some functions now test for NULL pointers early on to return TPM_RC_MEMORY. They would previously have failed in BigInitialized() if the BIGNUM had been a NULL pointer. Since TPM_RC_FAILURES should never occur, the new TPM_RC_MEMORYs should never occur, either. So, this is primarily a clean-up. Signed-off-by: Stefan Berger --- src/tpm2/TpmEcc_Signature_ECDSA.c | 6 ++++++ src/tpm2/crypto/openssl/CryptSym.c | 4 ++-- src/tpm2/crypto/openssl/Helpers.c | 13 ++++++++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/tpm2/TpmEcc_Signature_ECDSA.c b/src/tpm2/TpmEcc_Signature_ECDSA.c index 69cb7010..b8aaed5b 100644 --- a/src/tpm2/TpmEcc_Signature_ECDSA.c +++ b/src/tpm2/TpmEcc_Signature_ECDSA.c @@ -194,6 +194,9 @@ TpmEcc_SignEcdsa(Crypt_Int* bnR, // OUT: 'r' component of the signa const BIGNUM* s; BIGNUM* d = BN_new(); + if (!d) + return TPM_RC_MEMORY; + d = BigInitialized(d, (bigConst)bnD); eckey = EC_KEY_new(); @@ -310,6 +313,9 @@ TpmEcc_ValidateSignatureEcdsa( BIGNUM* s = BN_new(); EC_POINT* q = EcPointInitialized((bn_point_t*)ecQ, E); + if (!r || !s) + ERROR_EXIT(TPM_RC_MEMORY); + if (digest->b.size == CryptHashGetDigestSize(TPM_ALG_SHA1) && RuntimeProfileRequiresAttributeFlags(&g_RuntimeProfile, RUNTIME_ATTRIBUTE_NO_SHA1_VERIFICATION)) diff --git a/src/tpm2/crypto/openssl/CryptSym.c b/src/tpm2/crypto/openssl/CryptSym.c index 1ffae0cf..a0540258 100644 --- a/src/tpm2/crypto/openssl/CryptSym.c +++ b/src/tpm2/crypto/openssl/CryptSym.c @@ -591,7 +591,7 @@ CryptSymmetricEncrypt( buffersize = TPM2_ROUNDUP(dSize, blockSize); buffer = malloc(buffersize); if (buffer == NULL) - ERROR_EXIT(TPM_RC_FAILURE); + ERROR_EXIT(TPM_RC_MEMORY); pOut = buffer; } @@ -714,7 +714,7 @@ CryptSymmetricDecrypt( buffersize = TPM2_ROUNDUP(dSize + blockSize, blockSize); buffer = malloc(buffersize); if (buffer == NULL) - ERROR_EXIT(TPM_RC_FAILURE); + ERROR_EXIT(TPM_RC_MEMORY); #if ALG_TDES && ALG_CTR if (algorithm == TPM_ALG_TDES && mode == TPM_ALG_CTR) { diff --git a/src/tpm2/crypto/openssl/Helpers.c b/src/tpm2/crypto/openssl/Helpers.c index 308b70db..a22747ff 100644 --- a/src/tpm2/crypto/openssl/Helpers.c +++ b/src/tpm2/crypto/openssl/Helpers.c @@ -866,6 +866,9 @@ InitOpenSSLRSAPrivateKey(OBJECT *rsaKey, // IN BN_CTX *ctx = NULL; TPM_RC retVal; + if (!dP || !dQ || !qInv) + ERROR_EXIT(TPM_RC_MEMORY); + if (ObjectGetPublicParameters(rsaKey, &N, &E) != 1) ERROR_EXIT(TPM_RC_FAILURE); @@ -884,7 +887,7 @@ InitOpenSSLRSAPrivateKey(OBJECT *rsaKey, // IN Q = BN_new(); Qr = BN_new(); if (ctx == NULL || Q == NULL || Qr == NULL) - ERROR_EXIT(TPM_RC_FAILURE); + ERROR_EXIT(TPM_RC_MEMORY); /* Q = N/P; no remainder */ BN_set_flags(P, BN_FLG_CONSTTIME); // P is secret if (!BN_div(Q, Qr, N, P, ctx) || !BN_is_zero(Qr)) @@ -952,7 +955,9 @@ OpenSSLCryptRsaGenerateKey( EVP_PKEY *pkey = NULL; CRYPT_RSA_VAR(tmp); - if (bnE == NULL || BN_set_word(bnE, e) != 1) + if (bnE == NULL) + return TPM_RC_MEMORY; + if (BN_set_word(bnE, e) != 1) ERROR_EXIT(TPM_RC_FAILURE); if ((ctx = EVP_PKEY_CTX_new_from_name(NULL, "RSA", NULL)) == NULL || @@ -1016,7 +1021,9 @@ OpenSSLCryptRsaGenerateKey( BIGNUM *bnE = BN_new(); CRYPT_RSA_VAR(tmp); - if (bnE == NULL || BN_set_word(bnE, e) != 1) + if (bnE == NULL) + return TPM_RC_MEMORY; + if (BN_set_word(bnE, e) != 1) ERROR_EXIT(TPM_RC_FAILURE); rsa = RSA_new();