Pattern-Based Pre-filtering

The pattern filter module provides a fast pre-processing step that identifies common OCR errors using regular expressions before sending content to LLM analysis.

Core Implementation

defmodule Codexis.QualityChecker.PatternFilter do
  @moduledoc """
  Pre-filters content using known OCR error patterns before LLM processing
  """
  
  @known_patterns %{
    # Character substitutions
    ~r/\brn/ => "m",
    ~r/\bcl/ => "d",
    ~r/\bU\b/ => "ll",
    ~r/\btl1e\b/ => "the",
    ~r/\btl1at\b/ => "that",
    
    # Common word errors
    ~r/\bwith([a-z])/ => "with \\1",
    ~r/\b([a-z])of\b/ => "\\1 of",
    ~r/\b([a-z])the\b/ => "\\1 the",
    
    # Punctuation issues
    ~r/([a-z])\s*,([a-z])/ => "\\1, \\2",
    ~r/([a-z])\s*\.([a-z])/ => "\\1. \\2"
  }
  
  def pre_filter(text) do
    potential_errors = find_pattern_matches(text)
    
    %{
      text: text,
      potential_errors: potential_errors,
      needs_llm_check: length(potential_errors) > 0
    }
  end
  
  defp find_pattern_matches(text) do
    Enum.flat_map(@known_patterns, fn {pattern, suggestion} ->
      case Regex.scan(pattern, text, return: :index) do
        [] -> []
        matches ->
          Enum.map(matches, fn [{start, length}] ->
            %{
              position: start,
              length: length,
              original: String.slice(text, start, length),
              suggestion: apply_suggestion(text, start, length, pattern, suggestion),
              pattern: inspect(pattern)
            }
          end)
      end
    end)
  end
  
  defp apply_suggestion(text, start, length, pattern, suggestion) do
    matched_text = String.slice(text, start, length)
    Regex.replace(pattern, matched_text, suggestion)
  end
end

Extended Pattern Library

defmodule Codexis.QualityChecker.ExtendedPatterns do
  @moduledoc """
  Comprehensive pattern library for OCR error detection
  """
  
  @character_substitutions %{
    # Common character confusion
    ~r/rn/ => "m",
    ~r/cl/ => "d",
    ~r/li/ => "h",
    ~r/vv/ => "w",
    ~r/ii/ => "u",
    
    # Number-letter confusion
    ~r/\b0(?=[a-z])/i => "O",
    ~r/\b(?<=[a-z])0\b/i => "O",
    ~r/\b1(?=[a-z])/i => "l",
    ~r/\b(?<=[a-z])1\b/i => "l",
    ~r/\b5(?=[a-z])/i => "S",
    ~r/\b(?<=[a-z])5\b/i => "S",
    
    # Special character errors
    ~r/\-\-/ => "—",  # double hyphen to em-dash
    ~r/``/ => """,    # opening quotes
    ~r/''/ => """     # closing quotes
  }
  
  @word_errors %{
    # Common word mergers
    ~r/\bofthe\b/ => "of the",
    ~r/\bandthe\b/ => "and the",
    ~r/\binthe\b/ => "in the",
    ~r/\btothe\b/ => "to the",
    ~r/\bforthe\b/ => "for the",
    
    # Common splits
    ~r/\bwith\s+out\b/ => "without",
    ~r/\bsome\s+thing\b/ => "something",
    ~r/\bevery\s+thing\b/ => "everything",
    ~r/\bany\s+thing\b/ => "anything",
    
    # Specific common errors
    ~r/\btlie\b/ => "the",
    ~r/\btbe\b/ => "the",
    ~r/\bwbich\b/ => "which",
    ~r/\bwbere\b/ => "where",
    ~r/\bcornmon\b/ => "common",
    ~r/\brnany\b/ => "many",
    ~r/\brnight\b/ => "might"
  }
  
  @punctuation_errors %{
    # Missing spaces after punctuation
    ~r/([.,:;!?])([A-Z])/ => "\\1 \\2",
    
    # Extra spaces before punctuation
    ~r/\s+([.,:;!?])/ => "\\1",
    
    # Quote errors
    ~r/"([^"]+)"/ => ""\\1"",
    ~r/'([^']+)'/ => "'\\1'",
    
    # Apostrophe errors
    ~r/\b(\w+)'(\w+)\b/ => "\\1'\\2"
  }
  
  def get_all_patterns do
    Map.merge(@character_substitutions, 
      Map.merge(@word_errors, @punctuation_errors))
  end
  
  def get_pattern_category(pattern) do
    cond do
      Map.has_key?(@character_substitutions, pattern) -> :character
      Map.has_key?(@word_errors, pattern) -> :word
      Map.has_key?(@punctuation_errors, pattern) -> :punctuation
      true -> :unknown
    end
  end
end

Context-Aware Pattern Matching

defmodule Codexis.QualityChecker.ContextAwarePattern do
  @moduledoc """
  Pattern matching with context awareness
  """
  
  def find_contextual_errors(text) do
    words = String.split(text, ~r/\s+/)
    
    words
    |> Enum.with_index()
    |> Enum.flat_map(fn {word, index} ->
      check_word_in_context(word, index, words)
    end)
  end
  
  defp check_word_in_context(word, index, words) do
    prev_word = if index > 0, do: Enum.at(words, index - 1), else: nil
    next_word = if index < length(words) - 1, do: Enum.at(words, index + 1), else: nil
    
    errors = []
    
    # Check for article errors based on next word
    errors = errors ++ check_article_errors(word, next_word)
    
    # Check for preposition errors
    errors = errors ++ check_preposition_errors(word, prev_word, next_word)
    
    # Check for common phrase errors
    errors = errors ++ check_phrase_errors(word, prev_word, next_word)
    
    errors
  end
  
  defp check_article_errors("a", next_word) when next_word != nil do
    if String.match?(next_word, ~r/^[aeiou]/i) do
      [%{
        type: :article_error,
        original: "a",
        suggestion: "an",
        reason: "Use 'an' before vowel sounds"
      }]
    else
      []
    end
  end
  
  defp check_article_errors("an", next_word) when next_word != nil do
    if String.match?(next_word, ~r/^[^aeiou]/i) do
      [%{
        type: :article_error,
        original: "an",
        suggestion: "a",
        reason: "Use 'a' before consonant sounds"
      }]
    else
      []
    end
  end
  
  defp check_article_errors(_, _), do: []
  
  defp check_preposition_errors(word, prev_word, next_word) do
    case {prev_word, word, next_word} do
      {_, "if", "the"} -> []
      {_, "if", _} when next_word != nil ->
        [%{
          type: :preposition_error,
          original: "if",
          suggestion: "in",
          reason: "Possible OCR error: 'if' should be 'in'"
        }]
      _ -> []
    end
  end
  
  defp check_phrase_errors(word, prev_word, next_word) do
    case {prev_word, word, next_word} do
      {"as", "weil", "as"} ->
        [%{
          type: :phrase_error,
          original: "weil",
          suggestion: "well",
          reason: "Common phrase: 'as well as'"
        }]
      {"in", "tbe", _} ->
        [%{
          type: :phrase_error,
          original: "tbe",
          suggestion: "the",
          reason: "Common phrase: 'in the'"
        }]
      _ -> []
    end
  end
end

Pattern Learning System

defmodule Codexis.QualityChecker.PatternLearning do
  @moduledoc """
  Learns new patterns from confirmed corrections
  """
  
  use Agent
  
  def start_link(opts \\ []) do
    Agent.start_link(fn -> %{patterns: %{}, statistics: %{}} end, name: __MODULE__)
  end
  
  def learn_pattern(original, corrected, context \\ %{}) do
    Agent.update(__MODULE__, fn state ->
      pattern_key = {original, corrected}
      
      updated_patterns = Map.update(state.patterns, pattern_key, 
        %{count: 1, contexts: [context]},
        fn existing ->
          %{
            count: existing.count + 1,
            contexts: [context | existing.contexts]
          }
        end
      )
      
      %{state | patterns: updated_patterns}
    end)
  end
  
  def get_learned_patterns(min_occurrences \\ 3) do
    Agent.get(__MODULE__, fn state ->
      state.patterns
      |> Enum.filter(fn {_, data} -> data.count >= min_occurrences end)
      |> Enum.map(fn {{original, corrected}, data} ->
        %{
          original: original,
          corrected: corrected,
          count: data.count,
          confidence: calculate_confidence(data)
        }
      end)
      |> Enum.sort_by(& &1.confidence, :desc)
    end)
  end
  
  def save_patterns(path) do
    patterns = Agent.get(__MODULE__, fn state -> state.patterns end)
    
    content = patterns
    |> Enum.map(fn {{orig, corr}, data} ->
      "#{orig} => #{corr} (#{data.count} occurrences)"
    end)
    |> Enum.join("\n")
    
    File.write!(path, content)
  end
  
  defp calculate_confidence(data) do
    base_confidence = min(1.0, data.count / 10.0)
    
    # Adjust based on context variety
    context_variety = length(Enum.uniq(data.contexts))
    variety_bonus = min(0.2, context_variety / 20.0)
    
    base_confidence + variety_bonus
  end
end

Performance Optimization

defmodule Codexis.QualityChecker.OptimizedPatternFilter do
  @moduledoc """
  Optimized pattern filtering for large documents
  """
  
  def filter_parallel(text, chunk_size \\ 5000) do
    chunks = chunk_text(text, chunk_size)
    
    chunks
    |> Task.async_stream(&filter_chunk/1, max_concurrency: System.schedulers_online())
    |> Enum.reduce([], fn {:ok, results}, acc ->
      acc ++ results
    end)
    |> deduplicate_results()
  end
  
  defp chunk_text(text, size) do
    text
    |> String.graphemes()
    |> Enum.chunk_every(size)
    |> Enum.map(&Enum.join/1)
  end
  
  defp filter_chunk(chunk) do
    Codexis.QualityChecker.PatternFilter.pre_filter(chunk).potential_errors
  end
  
  defp deduplicate_results(results) do
    results
    |> Enum.uniq_by(fn result ->
      {result.position, result.original, result.suggestion}
    end)
    |> Enum.sort_by(& &1.position)
  end
  
  def compile_patterns(patterns) do
    # Pre-compile regex patterns for better performance
    Enum.map(patterns, fn {pattern, replacement} ->
      {Regex.compile!(pattern), replacement}
    end)
  end
  
  def benchmark_patterns(text, patterns) do
    Enum.map(patterns, fn {pattern, _} ->
      {time, matches} = :timer.tc(fn ->
        Regex.scan(pattern, text)
      end)
      
      %{
        pattern: inspect(pattern),
        time_us: time,
        matches: length(matches)
      }
    end)
  end
end