feat: Read ID cards (#571)

seshanthS · web-flow · commit 3a60cde9091f · 2025-05-27T14:25:22.000+02:00
diff --git a/app/ios/CameraView.swift b/app/ios/CameraView.swift
@@ -1,30 +1,37 @@
 // CameraView.swift
 // SwiftUI camera preview with frame capture callback
 
+import UIKit
 import SwiftUI
 import AVFoundation
 
 struct CameraView: UIViewControllerRepresentable {
-    var frameHandler: (UIImage) -> Void
+    var frameHandler: (UIImage, CGRect) -> Void
     var captureInterval: TimeInterval = 0.5 // seconds
+    var showOverlay: Bool = true  // For debug purposes. Set this value in LiveMRZScannerView.swift
 
     func makeUIViewController(context: Context) -> CameraViewController {
         let controller = CameraViewController()
         controller.frameHandler = frameHandler
         controller.captureInterval = captureInterval
+        controller.showOverlay = showOverlay
         return controller
     }
 
-    func updateUIViewController(_ uiViewController: CameraViewController, context: Context) {}
+    func updateUIViewController(_ uiViewController: CameraViewController, context: Context) {
+        uiViewController.showOverlay = showOverlay
+    }
 }
 
 class CameraViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate {
-    var frameHandler: ((UIImage) -> Void)?
+    var frameHandler: ((UIImage, CGRect) -> Void)?
     var captureInterval: TimeInterval = 0.5
+    var showOverlay: Bool = false
     private let session = AVCaptureSession()
     private let videoOutput = AVCaptureVideoDataOutput()
     private var lastCaptureTime = Date(timeIntervalSince1970: 0)
     private var previewLayer: AVCaptureVideoPreviewLayer?
+    private var roiOverlay: UIView? = nil
 
     override func viewDidLoad() {
         super.viewDidLoad()
@@ -45,12 +52,81 @@ class CameraViewController: UIViewController, AVCaptureVideoDataOutputSampleBuff
         if let previewLayer = previewLayer {
             view.layer.addSublayer(previewLayer)
         }
+        // ROI overlay - for debugging
+        if showOverlay && roiOverlay == nil {
+            let overlay = UIView()
+            overlay.layer.borderColor = UIColor.green.cgColor
+            overlay.layer.borderWidth = 2.0
+            overlay.backgroundColor = UIColor.clear
+            overlay.isUserInteractionEnabled = false
+            view.addSubview(overlay)
+            roiOverlay = overlay
+        }
         session.startRunning()
     }
 
+    private func calculateGreenBoxFrame() -> CGRect {
+        guard let previewLayer = previewLayer else { return .zero }
+        let videoRect = previewLayer.layerRectConverted(fromMetadataOutputRect: CGRect(x: 0, y: 0, width: 1, height: 1))
+        let visibleRect = videoRect.intersection(view.bounds)
+        
+        //Lottie animation frame
+        let lottieWidth = visibleRect.width * 1.3  // 130% of width
+        let lottieHeight = visibleRect.height * 1.3  // 130% of height
+        
+        //bottom 25% of the Lottie animation
+        let boxHeight = lottieHeight * 0.25
+        
+        // Center the box horizontally and ensure it's within bounds
+        let boxX = max(0, (visibleRect.width - lottieWidth) / 2)
+        let boxWidth = min(lottieWidth, visibleRect.width)
+        
+        //Vertical offset to move the ROI a bit up. 15% in this case
+        let verticalOffset = visibleRect.height * 0.15
+        
+        //GreenBox should stay within the visible area
+        let maxY = visibleRect.maxY - verticalOffset
+        let minY = visibleRect.minY
+        let boxY = max(minY, min(maxY - boxHeight, maxY - boxHeight))
+        // let boxY = visibleRect.maxY - boxHeight
+        
+        return CGRect(x: boxX, y: boxY, width: boxWidth, height: boxHeight)
+    }
+
+    var roiInImageCoordinates: CGRect {
+        guard let previewLayer = previewLayer else { return .zero }
+        let videoRect = previewLayer.layerRectConverted(fromMetadataOutputRect: CGRect(x: 0, y: 0, width: 1, height: 1))
+        let greenBox = calculateGreenBoxFrame()
+        
+        // map greenBox to normalized coordinates within videoRect
+        let normX = (greenBox.minX - videoRect.minX) / videoRect.width
+        let normY = (greenBox.minY - videoRect.minY) / videoRect.height
+        let normWidth = greenBox.width / videoRect.width
+        let normHeight = greenBox.height / videoRect.height
+        
+        // Ensure normalized coordinates are within [0,1] bounds as vision's max ROI is (0,0) to (1,1)
+        let clampedX = max(0, min(1, normX))
+        let clampedY = max(0, min(1, normY))
+        let clampedWidth = max(0, min(1 - clampedX, normWidth))
+        let clampedHeight = max(0, min(1 - clampedY, normHeight))
+        
+        // Vision expects (0,0) at bottom-left, so flip Y
+        let roiYVision = 1.0 - clampedY - clampedHeight
+        let roi = CGRect(x: clampedX, y: roiYVision, width: clampedWidth, height: clampedHeight)
+        
+        print("[CameraViewController] FINAL ROI for Vision (flipped Y, visible only): \(roi)")
+        return roi
+    }
+
     override func viewDidLayoutSubviews() {
         super.viewDidLayoutSubviews()
+        // Ensure previewLayer matches the visible area
         previewLayer?.frame = view.bounds
+        print("[CameraViewController] view.bounds: \(view.bounds)")
+        if let overlay = roiOverlay {
+            overlay.isHidden = !showOverlay
+            overlay.frame = calculateGreenBoxFrame()
+        }
     }
 
     func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
@@ -62,10 +138,11 @@ class CameraViewController: UIViewController, AVCaptureVideoDataOutputSampleBuff
         let context = CIContext()
         if let cgImage = context.createCGImage(ciImage, from: ciImage.extent) {
             let originalImage = UIImage(cgImage: cgImage, scale: UIScreen.main.scale, orientation: .right)
-            // Rotate to .up orientation
             let uprightImage = originalImage.fixedOrientation()
+            print("[CameraViewController] cgImage size: \(cgImage.width)x\(cgImage.height), preview size: \(view.bounds.size), orientation: \(uprightImage.imageOrientation.rawValue)")
+            let roi = roiInImageCoordinates
             DispatchQueue.main.async { [weak self] in
-                self?.frameHandler?(uprightImage)
+                self?.frameHandler?(uprightImage, roi)
             }
         }
     }
diff --git a/app/ios/LiveMRZScannerView.swift b/app/ios/LiveMRZScannerView.swift
@@ -80,38 +80,41 @@ struct LiveMRZScannerView: View {
 
     var body: some View {
         ZStack(alignment: .bottom) {
-                CameraView { image in
-                    // print("[LiveMRZScannerView] CameraView frame received. Size: \(image.size), Orientation: \(image.imageOrientation.rawValue)")
-                    if scanComplete { return }
-                    MRZScanner.scan(image: image) { result, boxes in
-                        recognizedText = result
-                        lastMRZDetection = Date()
-                        let parser = QKMRZParser(ocrCorrection: false)
-                        if let mrzResult = parser.parse(mrzString: result) {
-                            let doc = mrzResult;
-                            if doc.allCheckDigitsValid == true && !scanComplete {
-                                parsedMRZ = mrzResult
-                                scanComplete = true
-                                onScanComplete?(mrzResult)
-                                onScanResultAsDict?(mapVisionResultToDictionary(mrzResult))
-                            } else if doc.isDocumentNumberValid == false && !scanComplete {
-                                if let correctedResult = singleCorrectDocumentNumberInMRZ(result: result, docNumber: doc.documentNumber, parser: parser) {
-                                    let correctedDoc = correctedResult
-                                    if correctedDoc.allCheckDigitsValid == true {
-                                        parsedMRZ = correctedResult
-                                        scanComplete = true
-                                        onScanComplete?(correctedResult)
-                                        onScanResultAsDict?(mapVisionResultToDictionary(correctedResult))
+                CameraView(
+                    frameHandler: { image, roi in
+                        if scanComplete { return }
+                        MRZScanner.scan(image: image, roi: roi) { result, boxes in
+                            recognizedText = result
+                            lastMRZDetection = Date()
+                            // print("[LiveMRZScannerView] result: \(result)")
+                            let parser = QKMRZParser(ocrCorrection: false)
+                            if let mrzResult = parser.parse(mrzString: result) {
+                                let doc = mrzResult;
+                                if doc.allCheckDigitsValid == true && !scanComplete {
+                                    parsedMRZ = mrzResult
+                                    scanComplete = true
+                                    onScanComplete?(mrzResult)
+                                    onScanResultAsDict?(mapVisionResultToDictionary(mrzResult))
+                                } else if doc.isDocumentNumberValid == false && !scanComplete {
+                                    if let correctedResult = singleCorrectDocumentNumberInMRZ(result: result, docNumber: doc.documentNumber, parser: parser) {
+                                        let correctedDoc = correctedResult
+                                        if correctedDoc.allCheckDigitsValid == true {
+                                            parsedMRZ = correctedResult
+                                            scanComplete = true
+                                            onScanComplete?(correctedResult)
+                                            onScanResultAsDict?(mapVisionResultToDictionary(correctedResult))
+                                        }
                                     }
                                 }
-                            }
-                        } else {
-                            if !scanComplete {
-                                parsedMRZ = nil
+                            } else {
+                                if !scanComplete {
+                                    parsedMRZ = nil
+                                }
                             }
                         }
-                    }
-                }
+                    },
+                    showOverlay: false
+                )
 
             VStack {
                 if !scanComplete {
diff --git a/app/ios/MRZScanner.swift b/app/ios/MRZScanner.swift
@@ -5,47 +5,95 @@ import Vision
 import UIKit
 
 struct MRZScanner {
-    static func scan(image: UIImage, completion: @escaping (String, [CGRect]) -> Void) {
+    static func scan(image: UIImage, roi: CGRect? = nil, completion: @escaping (String, [CGRect]) -> Void) {
         guard let cgImage = image.cgImage else {
             completion("Image not valid", [])
             return
         }
 
         let request = VNRecognizeTextRequest { (request, error) in
+            if let error = error {
+                print("Vision error: \(error)")
+            }
+            
             guard let observations = request.results as? [VNRecognizedTextObservation] else {
+                print("No text observations found")
                 completion("No text found", [])
                 return
             }
 
+            // print("Found \(observations.count) text observations")
+            
             var mrzLines: [String] = []
             var boxes: [CGRect] = []
-            for obs in observations {
-                //   if let text = obs.topCandidates(1).first?.string, text.contains("<") {
-                    // mrzLines.append(text)
-                if let candidate = obs.topCandidates(1).first, candidate.string.contains("<") {
-                    mrzLines.append(candidate.string)
-                    boxes.append(obs.boundingBox) // Normalized coordinates
-                    // Log confidence for each character
-//                    for (i, char) in candidate.string.enumerated() {
-//                        if let box = try? candidate.boundingBox(for: candidate.string.index(candidate.string.startIndex, offsetBy: i)..<candidate.string.index(candidate.string.startIndex, offsetBy: i+1)) {
-//                            print("Char: \(char), Confidence: \(box.confidence)")
-//                        }
-//                    }
+            
+            // Sort lines from top to bottom
+            let sortedObservations = observations.sorted { $0.boundingBox.minY > $1.boundingBox.minY }
+            
+            for (index, obs) in sortedObservations.enumerated() {
+                if let candidate = obs.topCandidates(1).first {
+                    let text = candidate.string
+                    let confidence = candidate.confidence
+                    // print("Line \(index): '\(text)' (confidence: \(confidence), position: \(obs.boundingBox))")
+                    
+                    // Check if this looks like an MRZ line (either contains "<" or matches MRZ pattern)
+                    // TD1 format (ID cards): 30 chars, TD3 format (passports): 44 chars
+                    if text.contains("<") || 
+                       text.matches(pattern: "^[A-Z0-9<]{30}$") || //TD1 //case where there's no '<' in MRZ
+                       text.matches(pattern: "^[A-Z0-9<]{44}$")  //TD3
+                       {
+                        // print("Matched MRZ pattern: \(text)")
+                        mrzLines.append(text)
+                        boxes.append(obs.boundingBox)
+                        
+                        // Check if we have a complete MRZ
+                        if (mrzLines.count == 2 && mrzLines.allSatisfy { $0.count == 44 }) || // TD3 - passport
+                           (mrzLines.count == 3 && mrzLines.allSatisfy { $0.count == 30 }) {  // TD1 - ID card
+                            break
+                        }
+                    } else {
+                        print("Did not match MRZ pattern: \(text)")
+                    }
                 }
             }
 
             if mrzLines.isEmpty {
+                print("No MRZ lines found")
                 completion("", [])
             } else {
+                print("Found \(mrzLines.count) MRZ lines")
                 completion(mrzLines.joined(separator: "\n"), boxes)
             }
         }
         request.recognitionLevel = .accurate
         request.usesLanguageCorrection = false
         request.recognitionLanguages = ["en"]
+        
+        // Use provided ROI. If not use as bottom 20%
+        if let roi = roi {
+            print("[MRZScanner] Using provided ROI: \(roi) (image size: \(cgImage.width)x\(cgImage.height))")
+            request.regionOfInterest = roi
+        } else {
+            let imageHeight = CGFloat(cgImage.height)
+            let roiHeight = imageHeight * 0.2 // Bottom 20%
+            let defaultRoi = CGRect(x: 0, y: 0, width: 1.0, height: roiHeight / imageHeight)
+            print("[MRZScanner] Using default ROI: \(defaultRoi) (image size: \(cgImage.width)x\(cgImage.height), roi height: \(roiHeight))")
+            request.regionOfInterest = defaultRoi
+        }
+        
         let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
         DispatchQueue.global(qos: .userInitiated).async {
-            try? handler.perform([request])
+            do {
+                try handler.perform([request])
+            } catch {
+                print("Failed to perform recognition: \(error)")
+            }
         }
     }
 }
+
+extension String {
+    func matches(pattern: String) -> Bool {
+        return range(of: pattern, options: .regularExpression) != nil
+    }
+}
diff --git a/app/ios/PassportOCRViewManager.swift b/app/ios/PassportOCRViewManager.swift
@@ -37,7 +37,9 @@ class PassportOCRView: UIView {
                 "data": [
                   "documentNumber": resultDict["documentNumber"] as? String ?? "",
                   "expiryDate": resultDict["expiryDate"] as? String ?? "",
-                  "birthDate": resultDict["dateOfBirth"] as? String ?? ""
+                  "birthDate": resultDict["dateOfBirth"] as? String ?? "",
+                  "documentType": resultDict["documentType"] as? String ?? "",
+                  "countryCode": resultDict["countryCode"] as? String ?? ""
                 ]])
             }
         )
diff --git a/app/src/components/native/PassportCamera.tsx b/app/src/components/native/PassportCamera.tsx
@@ -18,6 +18,8 @@ interface RCTPassportOCRViewManagerProps extends RCTFragmentViewManagerProps {
             documentNumber: string;
             expiryDate: string;
             birthDate: string;
+            documentType: string;
+            countryCode: string;
           };
     }>,
   ) => void;
@@ -73,6 +75,8 @@ export const PassportCamera: React.FC<PassportCameraProps> = ({
               documentNumber: string;
               expiryDate: string;
               birthDate: string;
+              documentType: string;
+              countryCode: string;
             };
       }>,
     ) => {
@@ -86,6 +90,8 @@ export const PassportCamera: React.FC<PassportCameraProps> = ({
           passportNumber: event.nativeEvent.data.documentNumber,
           dateOfBirth: event.nativeEvent.data.birthDate,
           dateOfExpiry: event.nativeEvent.data.expiryDate,
+          documentType: event.nativeEvent.data.documentType,
+          countryCode: event.nativeEvent.data.countryCode,
         });
       }
     },
diff --git a/app/src/utils/utils.ts b/app/src/utils/utils.ts

Original file line number	Diff line number	Diff line change
`@@ -37,7 +37,9 @@ class PassportOCRView: UIView {`
`37`	`37`	`"data": [`
`38`	`38`	`"documentNumber": resultDict["documentNumber"] as? String ?? "",`
`39`	`39`	`"expiryDate": resultDict["expiryDate"] as? String ?? "",`
`40`		`- "birthDate": resultDict["dateOfBirth"] as? String ?? ""`
	`40`	`+ "birthDate": resultDict["dateOfBirth"] as? String ?? "",`
	`41`	`+ "documentType": resultDict["documentType"] as? String ?? "",`
	`42`	`+ "countryCode": resultDict["countryCode"] as? String ?? ""`
`41`	`43`	`]])`
`42`	`44`	`}`
`43`	`45`	`)`