Incorrect maximum encryption size for RSA OAEP

Sources/CryptoExtras/RSA/RSA.swift

@@ -508,7 +508,7 @@ extension _RSA.Encryption {
         /// - Warning: Key sizes less than 2048 are not recommended and should only be used for compatibility reasons.
         public init(unsafePEMRepresentation pemRepresentation: String) throws {
             self.backing = try BackingPublicKey(pemRepresentation: pemRepresentation)
-            guard self.keySizeInBits >= 2048, self.keySizeInBits % 8 == 0 else { throw CryptoKitError.incorrectParameterSize }
+            guard self.keySizeInBits >= 1024, self.keySizeInBits % 8 == 0 else { throw CryptoKitError.incorrectParameterSize }
         }
 
         /// Construct an RSA public key from a DER representation.
@@ -675,6 +675,14 @@ extension _RSA.Encryption {
     internal enum Digest {
         case sha1
         case sha256
+
+        /// Returns the number of bits in the resulting hash
+        var hashBitLength: Int {
+            switch self {
+            case .sha1: return 160
+            case .sha256: return 256
+            }
+        }
     }
 }
 
@@ -694,7 +702,7 @@ extension _RSA.Encryption.PublicKey {
     /// Return the maximum amount of data in bytes this key can encrypt in a single operation when using
     /// the specified padding mode.
     ///
-    /// ## Common values:
+    /// ## Common values (for PKCS1 OAEP SHA1):
     ///
     /// Key size|Padding|Max length
     /// -|-|-
@@ -703,8 +711,9 @@ extension _RSA.Encryption.PublicKey {
     /// 4096|PKCS-OAEP|470 bytes
     public func maximumEncryptSize(with padding: _RSA.Encryption.Padding) -> Int {
         switch padding.backing {
-        case .pkcs1_oaep:
-            return (self.keySizeInBits / 8) - 42
+        case let .pkcs1_oaep(Digest):
+            // https://datatracker.ietf.org/doc/html/rfc8017#section-7.1.1
+            return (self.keySizeInBits / 8) - (2 * Digest.hashBitLength / 8) - 2
         }
     }
 

Tests/CryptoExtrasTests/TestRSAEncryption.swift

@@ -140,6 +140,35 @@ final class TestRSAEncryption: XCTestCase {
             XCTAssertEqual(primitives.publicExponent, e)
         }
     }
+
+    func testMaximumEncryptSize() throws {
+        let pemRepresentation1024 = """
+            -----BEGIN PUBLIC KEY-----
+            MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCTLbu1QZhtXWKCHOjavP5NUCwJ
+            5DwjoMKGlEM/PQOMiY+wup8R1kCOHV6g+FvJ86laHJc0gqwFf1U51YxtQFy7cGV4
+            W2zJeTkqadO2fvTCjbZU+Oa78iVtTynq5h4yRWrTmveyzInhdVpi075Ql2hpGuET
+            H1qYVxqaDIJEHyETDQIDAQAB
+            -----END PUBLIC KEY-----
+            """
+        let pubKey1024 = try _RSA.Encryption.PublicKey(unsafePEMRepresentation: pemRepresentation1024)
+        XCTAssertEqual(86, pubKey1024.maximumEncryptSize(with: .PKCS1_OAEP))
+        XCTAssertEqual(62, pubKey1024.maximumEncryptSize(with: .PKCS1_OAEP_SHA256))
+
+        let pemRepresentation2048 = """
+            -----BEGIN PUBLIC KEY-----
+            MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx8IRKcs5FrHlWye2lfwc
+            Hr0Pi8g5iZhGMOOwmyIVsNULAvUIGZlIw38NNqebH3eF6ZxPiSRpwPwIs6QRcH5/
+            IwbHkUc0KdBbUwXDrLs0w00I7Flu5RP7IEfkOZdDGEWFY1pA3H1HaogxKFc5k3mM
+            s7pW6oty1eP4O7aVa/Pp363Vba7EZ2nru9lz4Ta+JU8UIHbpoddMGikGEKHrQ/Ge
+            n9RMNzSIy/e7TgTwC39GKn8fwN6VfcdNjvIhJrFNha/ORNArpzup7FUUauGLKt3a
+            jgsIjrAPBp63+Sy7+aFVoGTvI7DCkZ/Wv3JCFRuTAdYOa0A1xiqhTb1pcypvrd2T
+            ZQIDAQAB
+            -----END PUBLIC KEY-----
+            """
+        let pubKey2048 = try _RSA.Encryption.PublicKey(pemRepresentation: pemRepresentation2048)
+        XCTAssertEqual(214, pubKey2048.maximumEncryptSize(with: .PKCS1_OAEP))
+        XCTAssertEqual(190, pubKey2048.maximumEncryptSize(with: .PKCS1_OAEP_SHA256))
+    }
 }
 
 struct RSAEncryptionOAEPTestGroup: Codable {