Testing Guide

This guide covers testing strategies for the LLM-based scanno detection system in Codexis.

Test Structure

Unit Tests

defmodule Codexis.QualityChecker.ScannoDetectorTest do
  use ExUnit.Case, async: true
  
  alias Codexis.QualityChecker.ScannoDetector
  alias Codexis.LLM.MockClient
  
  describe "detect_scannos/2" do
    test "detects common OCR errors" do
      content = """
      The quickbrown fox jumps over tlie lazy dog. This is a 
      cornmon OCR error where "the" becomes "tlie" and words
      merge together. Also, "rn" often becomes "m" in words
      like "modern" which might appear as "modem".
      """
      
      {:ok, results} = ScannoDetector.detect_scannos(content, 
        model: MockClient,
        parallel: false
      )
      
      assert length(results.high) >= 2
      assert Enum.any?(results.high, fn c -> 
        c["original"] == "quickbrown" && c["corrected"] == "quick brown"
      end)
    end
    
    test "handles empty content" do
      {:ok, results} = ScannoDetector.detect_scannos("", model: MockClient)
      
      assert results.high == []
      assert results.medium == []
      assert results.low == []
    end
    
    test "respects confidence thresholds" do
      content = "Test content with errors"
      
      {:ok, results} = ScannoDetector.detect_scannos(content,
        model: MockClient,
        confidence_threshold: 0.9
      )
      
      assert Enum.all?(results.high, fn c -> c["confidence"] >= 0.9 end)
    end
  end
end

Integration Tests

defmodule Codexis.Integration.ScannoDetectionTest do
  use ExUnit.Case
  
  @moduletag :integration
  
  setup do
    # Start Ollama container for tests
    {:ok, _} = start_ollama_container()
    
    on_exit(fn ->
      stop_ollama_container()
    end)
    
    :ok
  end
  
  test "end-to-end scanno detection with real LLM" do
    manuscript_path = "test/fixtures/sample_manuscript.tex"
    
    {:ok, result} = Codexis.BuildManager.ScannoIntegration.check_manuscript_for_scannos(
      manuscript_path,
      interactive: false,
      model: "qwen2.5:3b"  # Use smaller model for tests
    )
    
    assert result.corrections_applied > 0
    assert File.exists?(result.output_path)
  end
  
  test "handles LaTeX-specific errors" do
    latex_content = """
    \\sectiori{Introduction}
    This is a test docurnent with LaTeX errors.
    \\cite{smith2O23}
    """
    
    {:ok, corrections} = ScannoDetector.detect_scannos(latex_content)
    
    assert Enum.any?(corrections.high, fn c ->
      c["original"] == "\\sectiori" && c["corrected"] == "\\section"
    end)
  end
end

Mock LLM Client

defmodule Codexis.LLM.MockClient do
  @behaviour Codexis.LLM.ClientBehaviour
  
  def generate(prompt, opts \\ []) do
    cond do
      String.contains?(prompt, "quickbrown") ->
        {:ok, quickbrown_response()}
      String.contains?(prompt, "tlie") ->
        {:ok, common_errors_response()}
      String.contains?(prompt, "\\sectiori") ->
        {:ok, latex_errors_response()}
      true ->
        {:ok, default_response()}
    end
  end
  
  defp quickbrown_response do
    """
    ```json
    {
      "corrections": [
        {
          "original": "quickbrown",
          "corrected": "quick brown",
          "reason": "Words appear merged",
          "confidence": 0.95,
          "position": 4
        }
      ]
    }
    ```
    """
  end
  
  defp common_errors_response do
    """
    ```json
    {
      "corrections": [
        {
          "original": "tlie",
          "corrected": "the",
          "reason": "Common OCR error: tlie→the",
          "confidence": 0.98,
          "position": 32
        },
        {
          "original": "cornmon",
          "corrected": "common",
          "reason": "Common OCR error: rn→m",
          "confidence": 0.92,
          "position": 54
        }
      ]
    }
    ```
    """
  end
  
  defp latex_errors_response do
    """
    ```json
    {
      "corrections": [
        {
          "original": "\\\\sectiori",
          "corrected": "\\\\section",
          "reason": "LaTeX command misspelling",
          "confidence": 0.96,
          "position": 0
        }
      ]
    }
    ```
    """
  end
  
  defp default_response do
    """
    ```json
    {
      "corrections": []
    }
    ```
    """
  end
end

Property-Based Tests

defmodule Codexis.Property.ScannoDetectorTest do
  use ExUnit.Case
  use ExUnitProperties
  
  property "never suggests empty corrections" do
    check all text <- string(:printable, min_length: 1) do
      {:ok, results} = ScannoDetector.detect_scannos(text, model: MockClient)
      
      all_corrections = results.high ++ results.medium ++ results.low
      
      assert Enum.all?(all_corrections, fn c ->
        String.trim(c["corrected"]) != ""
      end)
    end
  end
  
  property "correction positions are within text bounds" do
    check all text <- string(:printable, min_length: 10) do
      {:ok, results} = ScannoDetector.detect_scannos(text, model: MockClient)
      
      all_corrections = results.high ++ results.medium ++ results.low
      text_length = String.length(text)
      
      assert Enum.all?(all_corrections, fn c ->
        pos = c["absolute_position"]
        orig_length = String.length(c["original"])
        
        pos >= 0 && pos + orig_length <= text_length
      end)
    end
  end
  
  property "corrections preserve text structure" do
    check all text <- string(:printable, min_length: 5) do
      {:ok, results} = ScannoDetector.detect_scannos(text, model: MockClient)
      
      # Apply corrections shouldn't change overall structure drastically
      {:ok, corrected} = apply_corrections(text, results)
      
      # Same number of lines
      assert length(String.split(text, "\n")) == 
             length(String.split(corrected, "\n"))
    end
  end
end

Performance Tests

defmodule Codexis.Performance.ScannoDetectorTest do
  use ExUnit.Case
  
  @moduletag :performance
  
  test "processes large document within timeout" do
    large_document = generate_large_document(100_000)  # 100k characters
    
    {time, {:ok, _results}} = :timer.tc(fn ->
      ScannoDetector.detect_scannos(large_document,
        model: MockClient,
        parallel: true
      )
    end)
    
    # Should complete within 30 seconds
    assert time < 30_000_000
  end
  
  test "parallel processing is faster than sequential" do
    document = generate_large_document(50_000)
    
    {seq_time, _} = :timer.tc(fn ->
      ScannoDetector.detect_scannos(document, parallel: false)
    end)
    
    {par_time, _} = :timer.tc(fn ->
      ScannoDetector.detect_scannos(document, parallel: true)
    end)
    
    # Parallel should be at least 20% faster
    assert par_time < seq_time * 0.8
  end
  
  defp generate_large_document(size) do
    # Generate realistic document with known errors
    String.duplicate("This is a test docurnent with sorne errors. ", div(size, 40))
  end
end

Test Fixtures

defmodule Codexis.Test.Fixtures do
  def sample_manuscript do
    """
    \\documentclass{article}
    \\begin{document}
    
    \\sectiori{Introduction}
    This is a sarnple manuscript with OCR errors.
    The quickbrown fox jumps over tlie lazy dog.
    
    \\subsection{Cornmon Errors}
    Sorne common OCR errors include:
    - Character substitutions (rn→m)
    - Merged words (ofthe)
    - Number confusion (2O23 instead of 2023)
    
    \\end{document}
    """
  end
  
  def expected_corrections do
    [
      %{
        "original" => "\\sectiori",
        "corrected" => "\\section",
        "confidence" => 0.95
      },
      %{
        "original" => "sarnple",
        "corrected" => "sample",
        "confidence" => 0.92
      },
      %{
        "original" => "quickbrown",
        "corrected" => "quick brown",
        "confidence" => 0.90
      }
    ]
  end
  
  def mathematical_content do
    """
    \\[
    \\int_0^\\infty e^{-x^2} dx = \\frac{\\sqrt{\\pi}}{2}
    \\]
    
    The surn of the series is:
    \\[
    \\surn_{n=1}^\\infty \\frac{1}{n^2} = \\frac{\\pi^2}{6}
    \\]
    """
  end
end

UI Testing

defmodule Codexis.UI.ReviewInterfaceTest do
  use ExUnit.Case
  import ExUnit.CaptureIO
  
  alias Codexis.Review.InteractiveScannoReview
  
  test "displays correction with context" do
    correction = %{
      "original" => "tlie",
      "corrected" => "the",
      "absolute_position" => 20,
      "confidence" => 0.95,
      "reason" => "Common OCR error"
    }
    
    content = "This is a test with tlie error in it."
    
    output = capture_io(fn ->
      InteractiveScannoReview.display_correction(correction, content)
    end)
    
    assert output =~ "Original: tlie"
    assert output =~ "Suggested: the"
    assert output =~ "Confidence: 0.95"
  end
  
  test "handles user input correctly" do
    input = "a\n"  # Accept
    
    result = capture_io([input: input, capture_prompt: false], fn ->
      InteractiveScannoReview.get_user_decision()
    end)
    
    assert result == :accept
  end
end

Error Handling Tests

defmodule Codexis.ErrorHandling.ScannoDetectorTest do
  use ExUnit.Case
  
  test "handles LLM timeout gracefully" do
    slow_client = fn _prompt, _opts ->
      Process.sleep(1000)
      {:ok, "timeout"}
    end
    
    {:error, :timeout} = ScannoDetector.detect_scannos(
      "test content",
      model: slow_client,
      timeout: 500
    )
  end
  
  test "handles malformed LLM responses" do
    bad_response_client = fn _prompt, _opts ->
      {:ok, "not valid json"}
    end
    
    {:error, :invalid_response} = ScannoDetector.detect_scannos(
      "test content",
      model: bad_response_client
    )
  end
  
  test "recovers from partial failures in parallel processing" do
    flaky_client = fn prompt, _opts ->
      if String.contains?(prompt, "fail") do
        {:error, "simulated failure"}
      else
        {:ok, valid_response()}
      end
    end
    
    content = "Normal text. Text that will fail. More normal text."
    
    {:ok, results} = ScannoDetector.detect_scannos(
      content,
      model: flaky_client,
      parallel: true
    )
    
    # Should still get results from successful blocks
    assert length(results.high) > 0
  end
end

Test Helpers

defmodule Codexis.Test.Helpers do
  def with_temp_file(content, fun) do
    temp_file = Path.join(System.tmp_dir!(), "test_#{:rand.uniform(1000)}.tex")
    
    try do
      File.write!(temp_file, content)
      fun.(temp_file)
    after
      File.rm(temp_file)
    end
  end
  
  def assert_correction_valid(correction) do
    assert is_map(correction)
    assert Map.has_key?(correction, "original")
    assert Map.has_key?(correction, "corrected")
    assert Map.has_key?(correction, "confidence")
    assert correction["confidence"] >= 0 && correction["confidence"] <= 1
  end
  
  def count_corrections(results) do
    length(results.high) + length(results.medium) + length(results.low)
  end
  
  def apply_test_corrections(content, corrections) do
    # Apply corrections for testing
    Enum.reduce(corrections, content, fn correction, acc ->
      String.replace(acc, correction["original"], correction["corrected"])
    end)
  end
end

CI Test Configuration

# .github/workflows/test.yml
name: Test

on: [push, pull_request]

jobs:
  test:
    runs-on: ubuntu-latest
    services:
      ollama:
        image: ollama/ollama
        ports:
          - 11434:11434
    
    steps:
      - uses: actions/checkout@v2
      
      - name: Set up Elixir
        uses: erlef/setup-beam@v1
        with:
          elixir-version: '1.14'
          otp-version: '25'
      
      - name: Install dependencies
        run: mix deps.get
      
      - name: Pull test model
        run: |
          curl http://localhost:11434/api/pull -d '{"name":"qwen2.5:3b"}'
      
      - name: Run tests
        run: mix test --include integration
      
      - name: Run performance tests
        run: mix test --only performance

Test Configuration

# config/test.exs
config :codexis, :scanno_detection,
  enabled: true,
  default_model: "test_model",
  mock_llm: true,
  cache_directory: "test/support/cache",
  interactive: false,
  timeout: 5_000

# test/test_helper.exs
ExUnit.start(
  exclude: [:integration, :performance],
  capture_log: true
)

# Setup test directories
File.mkdir_p!("test/support/cache")

Acceptance Tests

defmodule Codexis.Acceptance.ScannoWorkflowTest do
  use ExUnit.Case
  
  @moduletag :acceptance
  
  test "complete workflow from detection to correction" do
    # 1. Create test manuscript with known errors
    manuscript = create_test_manuscript()
    
    # 2. Run scanno detection
    {:ok, corrections} = detect_scannos(manuscript)
    
    # 3. Simulate review process
    reviewed = review_corrections(corrections)
    
    # 4. Apply corrections
    {:ok, corrected} = apply_corrections(manuscript, reviewed)
    
    # 5. Verify results
    assert_corrections_applied(manuscript, corrected)
    assert_quality_improved(manuscript, corrected)
  end
  
  defp create_test_manuscript do
    """
    \\documentclass{article}
    \\title{Test Docurnent}
    \\author{John Srnith}
    
    \\begin{document}
    \\maketitle
    
    \\sectiori{Introduction}
    This docurnent has sorne OCR errors that need correctiori.
    
    \\end{document}
    """
  end
  
  defp assert_corrections_applied(original, corrected) do
    refute corrected =~ "docurnent"
    refute corrected =~ "sorne"
    refute corrected =~ "correctiori"
    refute corrected =~ "\\sectiori"
    refute corrected =~ "Srnith"
  end
  
  defp assert_quality_improved(original, corrected) do
    # Count remaining potential errors
    original_errors = count_potential_errors(original)
    corrected_errors = count_potential_errors(corrected)
    
    assert corrected_errors < original_errors * 0.5
  end
end

Coverage Reports

# mix.exs
def project do
  [
    app: :codexis,
    # ...
    test_coverage: [tool: ExCoveralls],
    preferred_cli_env: [
      coveralls: :test,
      "coveralls.detail": :test,
      "coveralls.post": :test,
      "coveralls.html": :test
    ]
  ]
end

# Run coverage
# mix coveralls.html

Benchmarking

defmodule Codexis.Benchmark do
  def run do
    content = File.read!("test/fixtures/large_manuscript.tex")
    
    Benchee.run(%{
      "sequential" => fn -> 
        ScannoDetector.detect_scannos(content, parallel: false)
      end,
      "parallel" => fn ->
        ScannoDetector.detect_scannos(content, parallel: true)
      end,
      "cached" => fn ->
        ScannoDetector.detect_scannos(content, use_cache: true)
      end
    })
  end
end

Test Best Practices

1. Test Isolation: Each test should be independent
2. Mock External Services: Use mocks for LLM calls in unit tests
3. Real Integration Tests: Have separate integration tests with real Ollama
4. Property-Based Testing: Use for edge cases and invariants
5. Performance Monitoring: Regular performance regression tests
6. CI/CD Integration: Automated testing in CI pipeline
7. Test Data Management: Maintain realistic test fixtures