Configure reporting with :raise and :log

Gemfile

@@ -23,4 +23,5 @@ gem "puma"
 gem "standard", "~> 1.12"
 gem "capybara-playwright-driver", require: "capybara/playwright"
 gem "cuprite", require: "capybara/cuprite"
+gem "rspec-matchers-active_support-notifications", github: "josegomezr/rspec-matchers-active_support-notifications"
 gem "selenium-webdriver"

README.md

@@ -89,6 +89,27 @@ the `payload` includes additional information:
 [ActiveSupport::OrderedOptions]: https://api.rubyonrails.org/classes/ActiveSupport/OrderedOptions.html
 [ActionDispatch::SystemTestCase]: https://api.rubyonrails.org/classes/ActionDispatch/SystemTestCase.html
 
+### `report.capybara_accessibility_audit` notification
+
+Subscribe to `report.capybara_accessibility_audit` notifications emitted when an
+accessibility audit detects violations. The `payload` includes:
+
+| Payload       | Type                               | Description |
+| ------------- | ---------------------------------- | ----------- |
+| report        | [Axe::API::Results][]              | The underlying axe.js [Results][] object
+| options       | [ActiveSupport::OrderedOptions][]  | The audit's configuration
+| test          | [ActionDispatch::SystemTestCase][] | The test case that triggered the audit
+
+[ActiveSupport::OrderedOptions]: https://api.rubyonrails.org/classes/ActiveSupport/OrderedOptions.html
+[ActionDispatch::SystemTestCase]: https://api.rubyonrails.org/classes/ActionDispatch/SystemTestCase.html
+[Axe::API::Results]: https://github.com/dequelabs/axe-core-gems/blob/v4.11.0/packages/axe-core-api/lib/axe/api/results.rb
+[Results]: https://www.deque.com/axe/core-documentation/api-documentation/#results-object
+
+> [!NOTE]
+> The `report.capybara_accessibility_audit` notifications are only published when
+> `config.capybara_accessibility_audit.reporter` is configured with
+> `:notification` or `:log`.
+
 ## Frequently Asked Questions
 
 My application already exists, automated accessibility audits are uncovering violations left and right. Do I have to fix them all at once?
@@ -143,6 +164,28 @@ end
 As you resolve the violations, you can remove entries from the list of skipped
 rules.
 
+How many I conduct a preliminary audit that comprehensively exercises my system test suite without failing tests that result in violations?
+---
+
+You can configure the audit's reporting mechanism. By default, the
+`config.capybara_accessibility_audit.reporter` value is set to `:raise`, which
+will raise violation errors that will fail the test suite.
+
+To log violations, rather than raise them, you can configure
+`config.capybara_accessibility_audit.reporter` to `:log`:
+
+
+```ruby
+class MySystemTest < ApplicationSystemTestCase
+  self.accessibility_audit_reporter = :log
+
+  test "with overridden accessibility :log reporter" do
+    visit examples_path
+    # ...
+  end
+end
+```
+
 I've implemented a custom Capybara action to toggle a disclosure element. How can I automatically audit for violations after it's called?
 ---
 

lib/capybara_accessibility_audit.rb

@@ -3,7 +3,20 @@
 loader.setup
 
 module CapybaraAccessibilityAudit
-  # Your code goes here...
+  extend self
+
+  def reporter_class(name)
+    case name
+    when :notification, :log
+      NotificationReporter
+    when :raise
+      RaiseReporter
+    when ::Class
+      name
+    else
+      raise ArgumentError.new("unsupported reporter: #{name}")
+    end
+  end
 end
 
 loader.eager_load

lib/capybara_accessibility_audit/audit_system_test_extensions.rb

@@ -13,6 +13,9 @@ module AuditSystemTestExtensions
       class_attribute :accessibility_audit_after_methods, default: Set.new
       class_attribute :accessibility_audit_enabled, default: true
       class_attribute :accessibility_audit_options, default: ActiveSupport::OrderedOptions.new
+      class_attribute :accessibility_audit_reporter, default: :raise
+
+      attr_accessor :accessibility_audit_auditor
 
       MODAL_METHODS.each do |method|
         define_method method do |*arguments, **options, &block|
@@ -87,8 +90,7 @@ def skip_accessibility_violations(value, &block)
       accessibility_audit_options.skipping = skipping
     end
 
-    def assert_no_accessibility_violations(auditor: accessibility_audit_options.auditor, **options)
-      options = options.with_defaults(accessibility_audit_options.except(:auditor))
+    def assert_no_accessibility_violations(auditor: accessibility_audit_auditor, **options)
       options.assert_valid_keys(
         :according_to,
         :checking,

lib/capybara_accessibility_audit/engine.rb

@@ -10,21 +10,22 @@ class Engine < ::Rails::Engine
       click_on
     ]
     config.capybara_accessibility_audit.audit_enabled = true
-    config.capybara_accessibility_audit.reporter = RaiseReporter
+    config.capybara_accessibility_audit.reporter = :raise
 
     initializer "capybara_accessibility_audit.minitest" do |app|
       ActiveSupport.on_load :action_dispatch_system_test_case do
         include CapybaraAccessibilityAudit::AuditSystemTestExtensions
 
         self.accessibility_audit_enabled = app.config.capybara_accessibility_audit.audit_enabled
+        self.accessibility_audit_reporter = app.config.capybara_accessibility_audit.reporter
 
         accessibility_audit_after app.config.capybara_accessibility_audit.audit_after
 
         setup do
           auditor_class = app.config.capybara_accessibility_audit.auditor
-          reporter_class = app.config.capybara_accessibility_audit.reporter
+          reporter_class = CapybaraAccessibilityAudit.reporter_class(accessibility_audit_reporter)
 
-          accessibility_audit_options.auditor = auditor_class.new(page, reporter_class.new(self))
+          self.accessibility_audit_auditor = auditor_class.new(page, reporter_class.new(self))
         end
       end
     end
@@ -43,9 +44,9 @@ class Engine < ::Rails::Engine
             accessibility_audit_after app.config.capybara_accessibility_audit.audit_after
 
             auditor_class = app.config.capybara_accessibility_audit.auditor
-            reporter_class = app.config.capybara_accessibility_audit.reporter
+            reporter_class = CapybaraAccessibilityAudit.reporter_class(accessibility_audit_reporter)
 
-            accessibility_audit_options.auditor = auditor_class.new(page, reporter_class.new(self))
+            self.accessibility_audit_auditor = auditor_class.new(page, reporter_class.new(self))
           end
 
           config.before(type: :system, &configure)

lib/capybara_accessibility_audit/log_subscriber.rb

@@ -0,0 +1,17 @@
+module CapybaraAccessibilityAudit
+  class LogSubscriber < ActiveSupport::LogSubscriber
+    def report(event)
+      report = event.payload[:report]
+      test = event.payload[:test]
+
+      if test.accessibility_audit_reporter == :log
+        error color(<<~ERROR, :red)
+          [capybara_accessibility_audit] Accessibility audit detected violations in "#{test.class.name}##{test.name}":
+          #{report.failure_message}
+        ERROR
+      end
+    end
+
+    attach_to :capybara_accessibility_audit
+  end
+end

lib/capybara_accessibility_audit/notification_reporter.rb

@@ -0,0 +1,24 @@
+module CapybaraAccessibilityAudit
+  class NotificationReporter
+    def initialize(test)
+      @test = test
+    end
+
+    def report(results)
+      if results.violations.present?
+        publish(results)
+      end
+    end
+
+    private
+
+    def publish(report)
+      ActiveSupport::Notifications.instrument "report.capybara_accessibility_audit", {
+        auditor: @test.accessibility_audit_auditor,
+        options: @test.accessibility_audit_options,
+        report: report,
+        test: @test
+      }
+    end
+  end
+end

spec/spec_helper.rb

@@ -7,10 +7,31 @@
 
 Dummy::Application.initialize!
 
+module ReporterSpecHelpers
+  def capture_report(&block)
+    assert_raises(Minitest::Assertion, &block).message
+  end
+
+  def assert_rule_violation(rule = nil, with: rule, without: nil, &block)
+    report = capture_report(&block)
+
+    Array(with).flatten.each do |included|
+      expect(report).to include(included)
+    end
+
+    Array(without).flatten.each do |excluded|
+      expect(report).not_to include(excluded)
+    end
+  end
+end
+
 RSpec.configure do |config|
   config.use_active_record = false
 
   config.filter_rails_from_backtrace!
+
+  config.include Rspec::Matchers::ActiveSupport::Notifications
+  config.include ReporterSpecHelpers, type: :system
 end
 
 Capybara.javascript_driver = ENV.fetch("DRIVER", "selenium_chrome_headless").to_sym

spec/system/notification_reporter_spec.rb

@@ -0,0 +1,63 @@
+require "spec_helper"
+
+RSpec.describe "Audit assertions", type: :system, js: true do
+  before :all do
+    driven_by Capybara.javascript_driver
+    self.accessibility_audit_reporter = :notification
+  end
+
+  it "does not report when there are no violations detected" do
+    expect do
+      visit violations_path
+
+      expect(page).to have_link("Violate rule: label")
+    end.to emit_event("report.capybara_accessibility_audit").exactly(0).times
+  end
+
+  it "reports on violations detected after #visit" do
+    assert_rule_violation "label: Form elements must have labels" do
+      visit violations_path(rules: %w[label])
+    end
+  end
+
+  it "reports on violations detected after #click_on" do
+    visit violations_path
+
+    assert_rule_violation "label: Form elements must have labels" do
+      click_on "Violate rule: label"
+    end
+  end
+
+  it "reports on violations detected after #click_link" do
+    visit violations_path
+
+    assert_rule_violation "label: Form elements must have labels" do
+      click_link "Violate rule: label"
+    end
+  end
+
+  describe "Skipping Audit After Method" do
+    skip_accessibility_audit_after :visit, :click_on
+
+    it "does not audit after a skipped method" do
+      visit violations_path
+      click_on "Violate rule: label"
+      go_back
+
+      assert_rule_violation("label: Form elements must have labels") do
+        click_link "Violate rule: label"
+      end
+    end
+  end
+
+  def capture_report(&block)
+    notification = nil
+
+    ActiveSupport::Notifications.subscribed(->(event) { notification = event }, "report.capybara_accessibility_audit", &block)
+
+    assert_equal self, notification.payload[:test]
+    assert_kind_of CapybaraAccessibilityAudit::AxeAuditor, notification.payload[:auditor]
+
+    notification.payload[:report].failure_message
+  end
+end

spec/system/audit_assertions_spec.rb → spec/system/raise_reporter_spec.rb

@@ -157,16 +157,4 @@
       click_on "Violate rule: label"
     end
   end
-
-  def assert_rule_violation(rule = nil, with: rule, without: nil, &block)
-    exception = assert_raises(Minitest::Assertion, &block)
-
-    Array(with).flatten.each do |included|
-      expect(exception.message).to include(included)
-    end
-
-    Array(without).flatten.each do |excluded|
-      expect(exception.message).not_to include(excluded)
-    end
-  end
 end

test/application_system_test_case.rb

@@ -5,15 +5,19 @@
 class ApplicationSystemTestCase < ActionDispatch::SystemTestCase
   driven_by Capybara.javascript_driver, screen_size: [1400, 1400], options: {js_errors: true}
 
+  def capture_report(&block)
+    assert_raises(Minitest::Assertion, &block).message
+  end
+
   def assert_rule_violation(rule = nil, with: rule, without: nil, &block)
-    exception = assert_raises(Minitest::Assertion, &block)
+    report = capture_report(&block)
 
     Array(with).flatten.each do |included|
-      assert_includes exception.message, included
+      assert_includes report, included
     end
 
     Array(without).flatten.each do |excluded|
-      assert_not_includes exception.message, excluded
+      assert_not_includes report, excluded
     end
   end
 end

test/system/audit_assertions_test.rb

@@ -90,12 +90,10 @@ class AuditAssertionsTest < ApplicationSystemTestCase
       end
     end
 
-    assert_kind_of CapybaraAccessibilityAudit::AxeAuditor, visit_audit.payload.dig(:options, :auditor)
     assert_equal self, click_on_audit.payload[:test]
     assert_equal :visit, visit_audit.payload[:method]
     assert_equal accessibility_audit_options, visit_audit.payload[:options]
 
-    assert_kind_of CapybaraAccessibilityAudit::AxeAuditor, click_on_audit.payload.dig(:options, :auditor)
     assert_equal self, click_on_audit.payload[:test]
     assert_equal :click_on, click_on_audit.payload[:method]
     assert_equal accessibility_audit_options, click_on_audit.payload[:options]

test/system/log_subscriber_test.rb

@@ -0,0 +1,53 @@
+require "application_system_test_case"
+require "active_support/log_subscriber/test_helper"
+
+module CapybaraAccessibilityAudit
+  class LogSubscriber::LogReporterTest < ApplicationSystemTestCase
+    include ActiveSupport::LogSubscriber::TestHelper
+
+    self.accessibility_audit_reporter = :log
+
+    def setup
+      super
+      LogSubscriber.attach_to :capybara_accessibility_audit
+    end
+
+    test "does not log reports without violations" do
+      visit violations_path
+
+      wait
+
+      assert_empty @logger.logged(:error)
+    end
+
+    test "logs violations detected after #click_on" do
+      visit violations_path
+      click_on "Violate rule: label"
+
+      wait
+
+      assert_equal 1, @logger.logged(:error).size
+      assert_match "[capybara_accessibility_audit]", @logger.logged(:error).first
+      assert_match "label: Form elements must have labels", @logger.logged(:error).first
+    end
+  end
+
+  class LogSubscriber::NotificationReporterTest < ApplicationSystemTestCase
+    include ActiveSupport::LogSubscriber::TestHelper
+
+    self.accessibility_audit_reporter = :notification
+
+    def setup
+      super
+      LogSubscriber.attach_to :capybara_accessibility_audit
+    end
+
+    test "does not logs violations detected" do
+      visit violations_path(rules: %w[label])
+
+      wait
+
+      assert_empty @logger.logged(:error)
+    end
+  end
+end