Add content negotiation test suite and fix map priority

Adds comprehensive CI test suite to verify content negotiation works correctly for all Accept header scenarios. The test suite validates that nginx serves markdown or HTML based on the Accept header, with proper fallback behavior.

Test coverage (16 test cases):
- Basic content negotiation (6 tests): text/markdown, application/markdown, text/plain, text/html, */*
- Browser behavior (2 tests): Complex Accept headers, HTML priority when listed first
- Direct access (2 tests): .md and .html file access
- Path variations (3 tests): Index paths, non-index paths, nested paths
- Edge cases (3 tests): 404 handling, fallback behavior, non-docs paths

Also fixes nginx map priority order to ensure anchored patterns (^text/html, ^text/plain) are evaluated before wildcard patterns. This ensures "text/html, text/markdown" correctly serves HTML instead of markdown.

Changes:
- Created bin/assert-content-negotiation.sh with run_test() helper function
- Integrated test into CircleCI test-nginx job
- Reordered nginx map patterns for correct priority matching
- All 16 tests passing locally

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

Author: kennethkalmer

.circleci/config.yml

@@ -128,6 +128,11 @@ jobs:
       - run:
           name: Verify all files are compressed
           command: ./bin/assert-compressed.sh
+      - run:
+          name: Test content negotiation for markdown
+          command: |
+            export PATH="$PWD/bin:$PWD/buildpack/build/.heroku-buildpack-nginx/ruby/bin:$PATH"
+            ./bin/assert-content-negotiation.sh
       - run:
           name: Test content request auth tokens
           command: |

bin/assert-content-negotiation.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+
+# Content Negotiation Test Suite
+# Verifies that nginx serves markdown or HTML based on Accept header
+
+source "$(dirname "$0")/nginx-utils.sh"
+trap stop_nginx EXIT
+
+set -euo pipefail
+
+# Disable auth for content negotiation tests
+export ENABLE_BASIC_AUTH=false
+export CONTENT_REQUEST_AUTH_TOKENS=""
+
+# Set default port if not already set
+export PORT=${PORT:-3001}
+
+# Test helper function
+# Parameters:
+#   $1: path - URL path to test
+#   $2: accept_header - Accept header value (empty string for default)
+#   $3: expected_status - Expected HTTP status code
+#   $4: expected_format - "html", "markdown", or "any"
+#   $5: test_name - Human-readable test description
+run_test() {
+  local path="$1"
+  local accept_header="$2"
+  local expected_status="$3"
+  local expected_format="$4"
+  local test_name="$5"
+
+  echo "🧪 $test_name"
+
+  # Build curl command with optional Accept header
+  local curl_cmd="curl --silent --header \"X-Forwarded-Proto: https\""
+
+  if [ -n "$accept_header" ]; then
+    curl_cmd="$curl_cmd --header \"Accept: $accept_header\""
+  fi
+
+  curl_cmd="$curl_cmd --write-out \"\\n%{http_code}\\n%{content_type}\""
+  curl_cmd="$curl_cmd \"http://localhost:\${PORT}\${path}\""
+
+  # Execute request and capture response + metadata
+  local response
+  response=$(eval "$curl_cmd")
+
+  # Parse response components
+  local body=$(echo "$response" | sed '$d' | sed '$d')
+  local status=$(echo "$response" | tail -2 | head -1)
+  local content_type=$(echo "$response" | tail -1)
+
+  # Assert status code
+  if [ "$status" != "$expected_status" ]; then
+    echo "  ❌ Expected status $expected_status, got $status"
+    exit 1
+  fi
+
+  # Verify content format
+  if [ "$expected_format" = "markdown" ]; then
+    # Check for markdown heading (first line should start with #)
+    local first_line=$(echo "$body" | head -1)
+    if ! grep -q "^#" <<< "$first_line"; then
+      echo "  ❌ Expected markdown (starting with #), got: ${first_line:0:50}"
+      exit 1
+    fi
+
+    # Verify Content-Type header (warning only, not fatal)
+    if ! grep -q "text/markdown" <<< "$content_type"; then
+      echo "  ⚠️  Warning: Content-Type is '$content_type', expected 'text/markdown'"
+    fi
+  elif [ "$expected_format" = "html" ]; then
+    # Check for HTML doctype using here-string to avoid broken pipe
+    if ! grep -q "<!DOCTYPE html>" <<< "$body"; then
+      echo "  ❌ Expected HTML (with DOCTYPE), but not found"
+      exit 1
+    fi
+  fi
+  # "any" format means we don't validate content
+
+  echo "  ✅ Passed (status: $status, format: $expected_format)"
+}
+
+# Main test suite
+echo "================================"
+echo "Content Negotiation Test Suite"
+echo "================================"
+echo
+
+start_nginx
+
+# Group 1: Basic Content Negotiation
+echo "Group 1: Basic Content Negotiation"
+echo "-----------------------------------"
+run_test "/docs/channels" "" "200" "html" "Default serves HTML"
+run_test "/docs/channels" "text/markdown" "200" "markdown" "Accept: text/markdown"
+run_test "/docs/channels" "application/markdown" "200" "markdown" "Accept: application/markdown"
+run_test "/docs/channels" "text/plain" "200" "markdown" "Accept: text/plain"
+run_test "/docs/channels" "text/html" "200" "html" "Accept: text/html"
+run_test "/docs/channels" "*/*" "200" "html" "Accept: */*"
+echo
+
+# Group 2: Browser Behavior
+echo "Group 2: Browser Behavior"
+echo "-------------------------"
+run_test "/docs/channels" "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" "200" "html" "Browser Accept header"
+run_test "/docs/channels" "text/html, text/markdown" "200" "html" "HTML prioritized when first"
+echo
+
+# Group 3: Direct Access
+echo "Group 3: Direct Access"
+echo "----------------------"
+run_test "/docs/channels.md" "" "200" "markdown" "Direct .md access"
+run_test "/docs/channels/index.html" "" "200" "html" "Direct index.html access"
+echo
+
+# Group 4: Path Variations
+echo "Group 4: Path Variations"
+echo "------------------------"
+run_test "/docs/chat/connect" "text/markdown" "200" "markdown" "Non-index path"
+run_test "/docs/api/realtime-sdk" "text/markdown" "200" "markdown" "Nested index path"
+run_test "/docs/basics" "text/markdown" "200" "markdown" "Simple path"
+echo
+
+# Group 5: Edge Cases
+echo "Group 5: Edge Cases"
+echo "-------------------"
+run_test "/docs/nonexistent" "" "404" "any" "404 when path missing"
+run_test "/docs/nonexistent" "text/markdown" "404" "any" "404 with markdown Accept"
+run_test "/llms.txt" "" "200" "any" "Non-docs paths unaffected"
+echo
+
+echo "================================"
+echo "✅ All 16 tests passed!"
+echo "================================"
+
+# Exit explicitly with success
+exit 0

config/nginx.conf.erb

@@ -74,13 +74,15 @@ http {
     "application/markdown"           ".md";
     "text/plain"                     ".md";
 
+    # IMPORTANT: Check start-of-string patterns FIRST (before wildcard patterns)
+    # Explicit HTML request gets HTML (handles browser defaults like "text/html, text/markdown")
+    "~*^text/html"                   ".html";
+    "~*^text/plain"                  ".md";
+
     # Handle multiple Accept values - prefer markdown if explicitly requested
     "~*text/markdown"                ".md";
     "~*application/markdown"         ".md";
-    "~*^text/plain"                  ".md";
 
-    # Explicit HTML request gets HTML (handles browser defaults)
-    "~*^text/html"                   ".html";
     "*/*"                            ".html";
   }