Merge pull request #530 from the-dan/fix-oaep-hash

Honor hash function set for RSA PKCS1 OAEP encryption
Yubico · Feb 3, 2025 · 3b1b91d · 3b1b91d
2 parents ab53895 + 720648b
commit 3b1b91d
Showing 1 changed file with 11 additions and 6 deletions.
diff --git a/ykcs11/openssl_utils.c b/ykcs11/openssl_utils.c
@@ -71,23 +71,28 @@ CK_RV do_rsa_encrypt(ykcs11_pkey_t *key, int padding, const ykcs11_md_t* oaep_md
     }
   }
 
-  if(oaep_md != NULL && oaep_mgf1 != NULL && oaep_label != NULL) {
-    if(EVP_PKEY_CTX_set_rsa_oaep_md(ctx, oaep_md) >= 0) {
+
+  if(oaep_md != NULL) {
+    if(EVP_PKEY_CTX_set_rsa_oaep_md(ctx, oaep_md) <= 0) {
       rv = CKR_FUNCTION_FAILED;
       goto rsa_enc_cleanup;
     }
-
-    if(EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, oaep_mgf1) >= 0) {
+  }
+
+  if (oaep_mgf1 != NULL) {
+    if(EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, oaep_mgf1) <= 0) {
       rv = CKR_FUNCTION_FAILED;
       goto rsa_enc_cleanup;
     }
+  }
 
-    if(EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, oaep_label, oaep_label_len) >= 0) {
+  if (oaep_label != NULL) {
+    if(EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, oaep_label, oaep_label_len) <= 0) {
       rv = CKR_FUNCTION_FAILED;
       goto rsa_enc_cleanup;
     }
   }
-
+   
   size_t cbLen = *enc_len;
   if(EVP_PKEY_encrypt(ctx, enc, &cbLen, data, data_len) <= 0) {
     rv = CKR_FUNCTION_FAILED;