Fix test_pkey_rsa.rb in FIPS.

* Fix test_new.
* Fix test_s_generate.
* Fix test_new_break.
* Fix test_sign_verify.
  Note that I created the signature text (`signature_encoded.txt`), that is used
  as a text to create the `signature0` in the `test_sign_verify` by the following
  steps with the `openssl` CLI on FIPS module.
  ```
  $ OPENSSL_DIR="${HOME}/.local/openssl-3.4.0-dev-fips-debug-3c6e114959"
  $ export OPENSSL_CONF="${OPENSSL_DIR}/ssl/openssl_fips.cnf"

  $ echo -n "Sign me!" > data.txt
  $ "${OPENSSL_DIR}/bin/openssl" dgst -sha256 -sign test/openssl/fixtures/pkey/rsa2048.pem data.txt > signature.txt
  $ cat signature.txt | base64 > signature_encoded.txt
  ```
* Fix test_sign_verify_options.
* Fix test_sign_verify_pss.

Author: junaruga

Rakefile

@@ -32,6 +32,7 @@ Rake::TestTask.new(:test_fips_internal) do |t|
     'test/openssl/test_pkey_dh.rb',
     'test/openssl/test_pkey_dsa.rb',
     'test/openssl/test_pkey_ec.rb',
+    'test/openssl/test_pkey_rsa.rb',
   ]
   t.warning = true
 end

test/openssl/test_pkey_rsa.rb

@@ -14,6 +14,20 @@ def test_no_private_exp
   end if !openssl?(3, 0, 0) # Impossible state in OpenSSL 3.0
 
   def test_private
+    # In FIPS, the `OpenSSL::PKey::RSA.new(2048, 65537)` with 2048 bits is at
+    # least required. However, generating a 2048-bit RSA key takes
+    # non-negligible amount of time. We want to avoid such slow tests as much
+    # as possible.
+    #
+    # SP800 requires ossl_ifc_ffc_compute_security_bits that the return value
+    # (strength in bits) is more than equal RSA_FIPS1864_MIN_KEYGEN_STRENGTH
+    # (112) in FIPS.
+    # https://github.com/openssl/openssl/blob/3c6e11495975a4eda4cc5886080afed6203711ac/crypto/rsa/rsa_sp800_56b_gen.c#L176-L182
+    # The ossl_ifc_ffc_compute_security_bits returns 112 with the argument nbits
+    # 2048.
+    # https://github.com/openssl/openssl/blob/3c6e11495975a4eda4cc5886080afed6203711ac/crypto/rsa/rsa_lib.c#L334-L335
+    omit_on_fips
+
     # Generated by key size and public exponent
     key = OpenSSL::PKey::RSA.new(512, 3)
     assert(key.private?)
@@ -46,63 +60,76 @@ def test_private
   end
 
   def test_new
-    key = OpenSSL::PKey::RSA.new(512)
-    assert_equal 512, key.n.num_bits
+    key = OpenSSL::PKey::RSA.new(2048)
+    assert_equal 2048, key.n.num_bits
     assert_equal 65537, key.e
     assert_not_nil key.d
 
-    # Specify public exponent
-    key2 = OpenSSL::PKey::RSA.new(512, 3)
-    assert_equal 512, key2.n.num_bits
-    assert_equal 3, key2.e
+    key2 = OpenSSL::PKey::RSA.new(2048, 65537)
+    assert_equal 2048, key2.n.num_bits
+    assert_equal 65537, key2.e
     assert_not_nil key2.d
   end
 
+  def test_new_exponent
+    # At least 2024-bits RSA key are required in FIPS.
+    omit_on_fips
+
+    # Specify public exponent
+    key = OpenSSL::PKey::RSA.new(512, 3)
+    assert_equal 512, key.n.num_bits
+    assert_equal 3, key.e
+  end
+
   def test_s_generate
-    key1 = OpenSSL::PKey::RSA.generate(512)
-    assert_equal 512, key1.n.num_bits
+    key1 = OpenSSL::PKey::RSA.generate(2048)
+    assert_equal 2048, key1.n.num_bits
     assert_equal 65537, key1.e
 
     # Specify public exponent
-    key2 = OpenSSL::PKey::RSA.generate(512, 3)
-    assert_equal 512, key2.n.num_bits
-    assert_equal 3, key2.e
+    key2 = OpenSSL::PKey::RSA.generate(2048, 65537)
+    assert_equal 2048, key2.n.num_bits
+    assert_equal 65537, key2.e
     assert_not_nil key2.d
   end
 
   def test_new_break
-    assert_nil(OpenSSL::PKey::RSA.new(1024) { break })
+    assert_nil(OpenSSL::PKey::RSA.new(2048) { break })
     assert_raise(RuntimeError) do
-      OpenSSL::PKey::RSA.new(1024) { raise }
+      OpenSSL::PKey::RSA.new(2048) { raise }
     end
   end
 
   def test_sign_verify
-    rsa1024 = Fixtures.pkey("rsa1024")
+    # Use 2024-bits RSA key, as OpenSSL 1.1.0 introduced that 512 or 1024-bits
+    # RSA key is insecure.
+    rsa = Fixtures.pkey("rsa2048")
     data = "Sign me!"
-    signature = rsa1024.sign("SHA256", data)
-    assert_equal true, rsa1024.verify("SHA256", signature, data)
+    signature = rsa.sign("SHA256", data)
+    assert_equal true, rsa.verify("SHA256", signature, data)
 
     signature0 = (<<~'end;').unpack1("m")
-      oLCgbprPvfhM4pjFQiDTFeWI9Sk+Og7Nh9TmIZ/xSxf2CGXQrptlwo7NQ28+
-      WA6YQo8jPH4hSuyWIM4Gz4qRYiYRkl5TDMUYob94zm8Si1HxEiS9354tzvqS
-      zS8MLW2BtNPuTubMxTItHGTnOzo9sUg0LAHVFt8kHG2NfKAw/gQ=
+      ooy49i8aeFtkDYUU0RPDsEugGiNw4lZxpbQPnIwtdftEkka945IqKZ/MY3YSw7wKsvBZeaTy8GqL
+      lSWLThsRFDV+UUS9zUBbQ9ygNIT8OjdV+tNL63ZpKGprczSnw4F05MQIpajNRud/8jiI9rf+Wysi
+      WwXecjMl2FlXlLJHY4PFQZU5TiametB4VCQRMcjLo1uf26u/yRpiGaYyqn5vxs0SqNtUDM1UL6x4
+      NHCAdqLjuFRQPjYp1vGLD3eSl4061pS8x1NVap3YGbYfGUyzZO4VfwFwf1jPdhp/OX/uZw4dGB2H
+      gSK+q1JiDFwEE6yym5tdKovL1g1NhFYHF6gkZg==
     end;
-    assert_equal true, rsa1024.verify("SHA256", signature0, data)
+    assert_equal true, rsa.verify("SHA256", signature0, data)
     signature1 = signature0.succ
-    assert_equal false, rsa1024.verify("SHA256", signature1, data)
+    assert_equal false, rsa.verify("SHA256", signature1, data)
   end
 
   def test_sign_verify_options
-    key = Fixtures.pkey("rsa1024")
+    key = Fixtures.pkey("rsa2048")
     data = "Sign me!"
     pssopts = {
       "rsa_padding_mode" => "pss",
       "rsa_pss_saltlen" => 20,
       "rsa_mgf1_md" => "SHA1"
     }
     sig_pss = key.sign("SHA256", data, pssopts)
-    assert_equal 128, sig_pss.bytesize
+    assert_equal 256, sig_pss.bytesize
     assert_equal true, key.verify("SHA256", sig_pss, data, pssopts)
     assert_equal true, key.verify_pss("SHA256", sig_pss, data,
                                       salt_length: 20, mgf1_hash: "SHA1")
@@ -175,12 +202,12 @@ def test_verify_empty_rsa
   end
 
   def test_sign_verify_pss
-    key = Fixtures.pkey("rsa1024")
+    key = Fixtures.pkey("rsa2048")
     data = "Sign me!"
     invalid_data = "Sign me?"
 
     signature = key.sign_pss("SHA256", data, salt_length: 20, mgf1_hash: "SHA1")
-    assert_equal 128, signature.bytesize
+    assert_equal 256, signature.bytesize
     assert_equal true,
       key.verify_pss("SHA256", signature, data, salt_length: 20, mgf1_hash: "SHA1")
     assert_equal true,
@@ -196,15 +223,23 @@ def test_sign_verify_pss
     assert_equal false,
       key.verify_pss("SHA256", signature, data, salt_length: 20, mgf1_hash: "SHA1")
 
-    signature = key.sign_pss("SHA256", data, salt_length: :max, mgf1_hash: "SHA1")
-    assert_equal true,
-      key.verify_pss("SHA256", signature, data, salt_length: 94, mgf1_hash: "SHA1")
-    assert_equal true,
-      key.verify_pss("SHA256", signature, data, salt_length: :auto, mgf1_hash: "SHA1")
-
-    assert_raise(OpenSSL::PKey::RSAError) {
-      key.sign_pss("SHA256", data, salt_length: 95, mgf1_hash: "SHA1")
-    }
+    # The sign_pss with `salt_length: :max` raises the "invalid salt length"
+    # error on the following part in FIPS. We need to skip the tests in FIPS.
+    # https://github.com/openssl/openssl/blob/d550d2aae531c6fa2e10b1a30d2acdf373663889/providers/implementations/signature/rsa_sig.c#L580-L597
+    unless OpenSSL.fips_mode
+      signature = key.sign_pss("SHA256", data, salt_length: :max, mgf1_hash: "SHA1")
+      # Should verify on the following salt_length (sLen).
+      # sLen <= emLen (octat) - 2 - hLen (octet) = 2048 / 8 - 2 - 256 / 8 = 222
+      # https://datatracker.ietf.org/doc/html/rfc8017#section-9.1.1
+      assert_equal true,
+        key.verify_pss("SHA256", signature, data, salt_length: 222, mgf1_hash: "SHA1")
+      assert_equal true,
+        key.verify_pss("SHA256", signature, data, salt_length: :auto, mgf1_hash: "SHA1")
+
+      assert_raise(OpenSSL::PKey::RSAError) {
+        key.sign_pss("SHA256", data, salt_length: 223, mgf1_hash: "SHA1")
+      }
+    end
   end
 
   def test_encrypt_decrypt