diff --git a/Gemfile b/Gemfile index 33164fa..5737212 100644 --- a/Gemfile +++ b/Gemfile @@ -23,4 +23,5 @@ gem "puma" gem "standard", "~> 1.12" gem "capybara-playwright-driver", require: "capybara/playwright" gem "cuprite", require: "capybara/cuprite" +gem "rspec-matchers-active_support-notifications", github: "josegomezr/rspec-matchers-active_support-notifications" gem "selenium-webdriver" diff --git a/README.md b/README.md index 10c23d8..e5609ca 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,27 @@ the `payload` includes additional information: [ActiveSupport::OrderedOptions]: https://api.rubyonrails.org/classes/ActiveSupport/OrderedOptions.html [ActionDispatch::SystemTestCase]: https://api.rubyonrails.org/classes/ActionDispatch/SystemTestCase.html +### `report.capybara_accessibility_audit` notification + +Subscribe to `report.capybara_accessibility_audit` notifications emitted when an +accessibility audit detects violations. The `payload` includes: + +| Payload | Type | Description | +| ------------- | ---------------------------------- | ----------- | +| report | [Axe::API::Results][] | The underlying axe.js [Results][] object +| options | [ActiveSupport::OrderedOptions][] | The audit's configuration +| test | [ActionDispatch::SystemTestCase][] | The test case that triggered the audit + +[ActiveSupport::OrderedOptions]: https://api.rubyonrails.org/classes/ActiveSupport/OrderedOptions.html +[ActionDispatch::SystemTestCase]: https://api.rubyonrails.org/classes/ActionDispatch/SystemTestCase.html +[Axe::API::Results]: https://github.com/dequelabs/axe-core-gems/blob/v4.11.0/packages/axe-core-api/lib/axe/api/results.rb +[Results]: https://www.deque.com/axe/core-documentation/api-documentation/#results-object + +> [!NOTE] +> The `report.capybara_accessibility_audit` notifications are only published when +> `config.capybara_accessibility_audit.reporter` is configured with +> `:notification` or `:log`. + ## Frequently Asked Questions My application already exists, automated accessibility audits are uncovering violations left and right. Do I have to fix them all at once? @@ -143,6 +164,28 @@ end As you resolve the violations, you can remove entries from the list of skipped rules. +How many I conduct a preliminary audit that comprehensively exercises my system test suite without failing tests that result in violations? +--- + +You can configure the audit's reporting mechanism. By default, the +`config.capybara_accessibility_audit.reporter` value is set to `:raise`, which +will raise violation errors that will fail the test suite. + +To log violations, rather than raise them, you can configure +`config.capybara_accessibility_audit.reporter` to `:log`: + + +```ruby +class MySystemTest < ApplicationSystemTestCase + self.accessibility_audit_reporter = :log + + test "with overridden accessibility :log reporter" do + visit examples_path + # ... + end +end +``` + I've implemented a custom Capybara action to toggle a disclosure element. How can I automatically audit for violations after it's called? --- diff --git a/lib/capybara_accessibility_audit.rb b/lib/capybara_accessibility_audit.rb index 9831378..2438d55 100644 --- a/lib/capybara_accessibility_audit.rb +++ b/lib/capybara_accessibility_audit.rb @@ -3,7 +3,20 @@ loader.setup module CapybaraAccessibilityAudit - # Your code goes here... + extend self + + def reporter_class(name) + case name + when :notification, :log + NotificationReporter + when :raise + RaiseReporter + when ::Class + name + else + raise ArgumentError.new("unsupported reporter: #{name}") + end + end end loader.eager_load diff --git a/lib/capybara_accessibility_audit/audit_system_test_extensions.rb b/lib/capybara_accessibility_audit/audit_system_test_extensions.rb index 97ad7ce..acad619 100644 --- a/lib/capybara_accessibility_audit/audit_system_test_extensions.rb +++ b/lib/capybara_accessibility_audit/audit_system_test_extensions.rb @@ -13,6 +13,9 @@ module AuditSystemTestExtensions class_attribute :accessibility_audit_after_methods, default: Set.new class_attribute :accessibility_audit_enabled, default: true class_attribute :accessibility_audit_options, default: ActiveSupport::OrderedOptions.new + class_attribute :accessibility_audit_reporter, default: :raise + + attr_accessor :accessibility_audit_auditor MODAL_METHODS.each do |method| define_method method do |*arguments, **options, &block| @@ -87,8 +90,7 @@ def skip_accessibility_violations(value, &block) accessibility_audit_options.skipping = skipping end - def assert_no_accessibility_violations(auditor: accessibility_audit_options.auditor, **options) - options = options.with_defaults(accessibility_audit_options.except(:auditor)) + def assert_no_accessibility_violations(auditor: accessibility_audit_auditor, **options) options.assert_valid_keys( :according_to, :checking, diff --git a/lib/capybara_accessibility_audit/engine.rb b/lib/capybara_accessibility_audit/engine.rb index 7a681e6..41adfdd 100644 --- a/lib/capybara_accessibility_audit/engine.rb +++ b/lib/capybara_accessibility_audit/engine.rb @@ -10,21 +10,22 @@ class Engine < ::Rails::Engine click_on ] config.capybara_accessibility_audit.audit_enabled = true - config.capybara_accessibility_audit.reporter = RaiseReporter + config.capybara_accessibility_audit.reporter = :raise initializer "capybara_accessibility_audit.minitest" do |app| ActiveSupport.on_load :action_dispatch_system_test_case do include CapybaraAccessibilityAudit::AuditSystemTestExtensions self.accessibility_audit_enabled = app.config.capybara_accessibility_audit.audit_enabled + self.accessibility_audit_reporter = app.config.capybara_accessibility_audit.reporter accessibility_audit_after app.config.capybara_accessibility_audit.audit_after setup do auditor_class = app.config.capybara_accessibility_audit.auditor - reporter_class = app.config.capybara_accessibility_audit.reporter + reporter_class = CapybaraAccessibilityAudit.reporter_class(accessibility_audit_reporter) - accessibility_audit_options.auditor = auditor_class.new(page, reporter_class.new(self)) + self.accessibility_audit_auditor = auditor_class.new(page, reporter_class.new(self)) end end end @@ -43,9 +44,9 @@ class Engine < ::Rails::Engine accessibility_audit_after app.config.capybara_accessibility_audit.audit_after auditor_class = app.config.capybara_accessibility_audit.auditor - reporter_class = app.config.capybara_accessibility_audit.reporter + reporter_class = CapybaraAccessibilityAudit.reporter_class(accessibility_audit_reporter) - accessibility_audit_options.auditor = auditor_class.new(page, reporter_class.new(self)) + self.accessibility_audit_auditor = auditor_class.new(page, reporter_class.new(self)) end config.before(type: :system, &configure) diff --git a/lib/capybara_accessibility_audit/log_subscriber.rb b/lib/capybara_accessibility_audit/log_subscriber.rb new file mode 100644 index 0000000..973c3e2 --- /dev/null +++ b/lib/capybara_accessibility_audit/log_subscriber.rb @@ -0,0 +1,17 @@ +module CapybaraAccessibilityAudit + class LogSubscriber < ActiveSupport::LogSubscriber + def report(event) + report = event.payload[:report] + test = event.payload[:test] + + if test.accessibility_audit_reporter == :log + error color(<<~ERROR, :red) + [capybara_accessibility_audit] Accessibility audit detected violations in "#{test.class.name}##{test.name}": + #{report.failure_message} + ERROR + end + end + + attach_to :capybara_accessibility_audit + end +end diff --git a/lib/capybara_accessibility_audit/notification_reporter.rb b/lib/capybara_accessibility_audit/notification_reporter.rb new file mode 100644 index 0000000..ef5597d --- /dev/null +++ b/lib/capybara_accessibility_audit/notification_reporter.rb @@ -0,0 +1,24 @@ +module CapybaraAccessibilityAudit + class NotificationReporter + def initialize(test) + @test = test + end + + def report(results) + if results.violations.present? + publish(results) + end + end + + private + + def publish(report) + ActiveSupport::Notifications.instrument "report.capybara_accessibility_audit", { + auditor: @test.accessibility_audit_auditor, + options: @test.accessibility_audit_options, + report: report, + test: @test + } + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index f955334..6cef997 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -7,10 +7,31 @@ Dummy::Application.initialize! +module ReporterSpecHelpers + def capture_report(&block) + assert_raises(Minitest::Assertion, &block).message + end + + def assert_rule_violation(rule = nil, with: rule, without: nil, &block) + report = capture_report(&block) + + Array(with).flatten.each do |included| + expect(report).to include(included) + end + + Array(without).flatten.each do |excluded| + expect(report).not_to include(excluded) + end + end +end + RSpec.configure do |config| config.use_active_record = false config.filter_rails_from_backtrace! + + config.include Rspec::Matchers::ActiveSupport::Notifications + config.include ReporterSpecHelpers, type: :system end Capybara.javascript_driver = ENV.fetch("DRIVER", "selenium_chrome_headless").to_sym diff --git a/spec/system/notification_reporter_spec.rb b/spec/system/notification_reporter_spec.rb new file mode 100644 index 0000000..d5e534e --- /dev/null +++ b/spec/system/notification_reporter_spec.rb @@ -0,0 +1,63 @@ +require "spec_helper" + +RSpec.describe "Audit assertions", type: :system, js: true do + before :all do + driven_by Capybara.javascript_driver + self.accessibility_audit_reporter = :notification + end + + it "does not report when there are no violations detected" do + expect do + visit violations_path + + expect(page).to have_link("Violate rule: label") + end.to emit_event("report.capybara_accessibility_audit").exactly(0).times + end + + it "reports on violations detected after #visit" do + assert_rule_violation "label: Form elements must have labels" do + visit violations_path(rules: %w[label]) + end + end + + it "reports on violations detected after #click_on" do + visit violations_path + + assert_rule_violation "label: Form elements must have labels" do + click_on "Violate rule: label" + end + end + + it "reports on violations detected after #click_link" do + visit violations_path + + assert_rule_violation "label: Form elements must have labels" do + click_link "Violate rule: label" + end + end + + describe "Skipping Audit After Method" do + skip_accessibility_audit_after :visit, :click_on + + it "does not audit after a skipped method" do + visit violations_path + click_on "Violate rule: label" + go_back + + assert_rule_violation("label: Form elements must have labels") do + click_link "Violate rule: label" + end + end + end + + def capture_report(&block) + notification = nil + + ActiveSupport::Notifications.subscribed(->(event) { notification = event }, "report.capybara_accessibility_audit", &block) + + assert_equal self, notification.payload[:test] + assert_kind_of CapybaraAccessibilityAudit::AxeAuditor, notification.payload[:auditor] + + notification.payload[:report].failure_message + end +end diff --git a/spec/system/audit_assertions_spec.rb b/spec/system/raise_reporter_spec.rb similarity index 92% rename from spec/system/audit_assertions_spec.rb rename to spec/system/raise_reporter_spec.rb index c11daec..6b3e496 100644 --- a/spec/system/audit_assertions_spec.rb +++ b/spec/system/raise_reporter_spec.rb @@ -157,16 +157,4 @@ click_on "Violate rule: label" end end - - def assert_rule_violation(rule = nil, with: rule, without: nil, &block) - exception = assert_raises(Minitest::Assertion, &block) - - Array(with).flatten.each do |included| - expect(exception.message).to include(included) - end - - Array(without).flatten.each do |excluded| - expect(exception.message).not_to include(excluded) - end - end end diff --git a/test/application_system_test_case.rb b/test/application_system_test_case.rb index 8e01644..a651022 100644 --- a/test/application_system_test_case.rb +++ b/test/application_system_test_case.rb @@ -5,15 +5,19 @@ class ApplicationSystemTestCase < ActionDispatch::SystemTestCase driven_by Capybara.javascript_driver, screen_size: [1400, 1400], options: {js_errors: true} + def capture_report(&block) + assert_raises(Minitest::Assertion, &block).message + end + def assert_rule_violation(rule = nil, with: rule, without: nil, &block) - exception = assert_raises(Minitest::Assertion, &block) + report = capture_report(&block) Array(with).flatten.each do |included| - assert_includes exception.message, included + assert_includes report, included end Array(without).flatten.each do |excluded| - assert_not_includes exception.message, excluded + assert_not_includes report, excluded end end end diff --git a/test/system/audit_assertions_test.rb b/test/system/audit_assertions_test.rb index 8ec47b4..84d969e 100644 --- a/test/system/audit_assertions_test.rb +++ b/test/system/audit_assertions_test.rb @@ -90,12 +90,10 @@ class AuditAssertionsTest < ApplicationSystemTestCase end end - assert_kind_of CapybaraAccessibilityAudit::AxeAuditor, visit_audit.payload.dig(:options, :auditor) assert_equal self, click_on_audit.payload[:test] assert_equal :visit, visit_audit.payload[:method] assert_equal accessibility_audit_options, visit_audit.payload[:options] - assert_kind_of CapybaraAccessibilityAudit::AxeAuditor, click_on_audit.payload.dig(:options, :auditor) assert_equal self, click_on_audit.payload[:test] assert_equal :click_on, click_on_audit.payload[:method] assert_equal accessibility_audit_options, click_on_audit.payload[:options] diff --git a/test/system/log_subscriber_test.rb b/test/system/log_subscriber_test.rb new file mode 100644 index 0000000..294cd93 --- /dev/null +++ b/test/system/log_subscriber_test.rb @@ -0,0 +1,53 @@ +require "application_system_test_case" +require "active_support/log_subscriber/test_helper" + +module CapybaraAccessibilityAudit + class LogSubscriber::LogReporterTest < ApplicationSystemTestCase + include ActiveSupport::LogSubscriber::TestHelper + + self.accessibility_audit_reporter = :log + + def setup + super + LogSubscriber.attach_to :capybara_accessibility_audit + end + + test "does not log reports without violations" do + visit violations_path + + wait + + assert_empty @logger.logged(:error) + end + + test "logs violations detected after #click_on" do + visit violations_path + click_on "Violate rule: label" + + wait + + assert_equal 1, @logger.logged(:error).size + assert_match "[capybara_accessibility_audit]", @logger.logged(:error).first + assert_match "label: Form elements must have labels", @logger.logged(:error).first + end + end + + class LogSubscriber::NotificationReporterTest < ApplicationSystemTestCase + include ActiveSupport::LogSubscriber::TestHelper + + self.accessibility_audit_reporter = :notification + + def setup + super + LogSubscriber.attach_to :capybara_accessibility_audit + end + + test "does not logs violations detected" do + visit violations_path(rules: %w[label]) + + wait + + assert_empty @logger.logged(:error) + end + end +end diff --git a/test/system/notification_reporter_test.rb b/test/system/notification_reporter_test.rb new file mode 100644 index 0000000..bebab35 --- /dev/null +++ b/test/system/notification_reporter_test.rb @@ -0,0 +1,62 @@ +require "application_system_test_case" + +module CapybaraAccessibilityAudit + class NotificationReporter::TestCase < ApplicationSystemTestCase + self.accessibility_audit_reporter = :notification + + def capture_report(&block) + notification = assert_notification("report.capybara_accessibility_audit", &block) + + assert_equal self, notification.payload[:test] + assert_kind_of AxeAuditor, notification.payload[:auditor] + + notification.payload[:report].failure_message + end + end + + class NotificationReporter::AuditAssertionsTest < NotificationReporter::TestCase + test "does not report when there are no violations detected" do + assert_no_notifications "report.capybara_accessibility_audit" do + visit violations_path + + assert_link "Violate rule: label" + end + end + + test "reports on violations detected after #visit" do + assert_rule_violation "label: Form elements must have labels" do + visit violations_path(rules: %w[label]) + end + end + + test "reports on violations detected after #click_on" do + visit violations_path + + assert_rule_violation "label: Form elements must have labels" do + click_on "Violate rule: label" + end + end + + test "reports on violations detected after #click_link" do + visit violations_path + + assert_rule_violation "label: Form elements must have labels" do + click_link "Violate rule: label" + end + end + end + + class NotificationReporter::SkippingAuditAfterMethodTest < NotificationReporter::TestCase + skip_accessibility_audit_after :visit, :click_on + + test "does not audit after a skipped method" do + visit violations_path + click_on "Violate rule: label" + go_back + + assert_rule_violation("label: Form elements must have labels") do + click_link "Violate rule: label" + end + end + end +end