Specialized Content Detectors

Specialized detectors handle specific types of content that require different processing approaches, such as LaTeX commands, mathematical equations, and bibliographic references.

Core Implementation

defmodule Codexis.QualityChecker.SpecializedDetectors do
  @moduledoc """
  Specialized scanno detection for different content types
  """
  
  alias Codexis.LLM.OllamaClient
  
  def check_latex_commands(commands, model \\ "qwen3.0:8b") do
    prompt = """
    Check these LaTeX commands for OCR errors. Common issues:
    - Misspelled command names (\\sectiori → \\section)
    - Wrong brackets (\\cite(ref) → \\cite{ref})
    - Invalid command syntax
    
    Commands to check:
    #{Enum.map(commands, & &1.full) |> Enum.join("\n")}
    
    List any corrections needed.
    """
    
    OllamaClient.generate(prompt, model: model)
  end
  
  def check_math_content(math_blocks, model \\ "deepseek-r1:8b") do
    # Use DeepSeek-R1 for better mathematical reasoning
    math_blocks
    |> Enum.map(fn block ->
      check_single_math_block(block, model)
    end)
  end
  
  defp check_single_math_block(block, model) do
    prompt = """
    <anki_thinking>
    I need to check this mathematical expression for OCR errors. Common issues:
    - Greek letters confused with similar Latin letters
    - Subscripts/superscripts misplaced
    - Operators confused (− vs -, × vs x)
    - Parentheses/brackets mixed up
    - Mathematical symbols corrupted
    </anki_thinking>
    
    Check this LaTeX math for OCR errors:
    
    #{block.content}
    
    Look for:
    1. Malformed mathematical notation
    2. Inconsistent variable names
    3. Unbalanced delimiters
    4. Invalid LaTeX math syntax
    
    Provide corrections if needed.
    """
    
    OllamaClient.generate(prompt, model: model, temperature: 0.1)
  end
  
  def check_references(citations, model \\ "qwen3.0:8b") do
    prompt = """
    Check these bibliography citations for OCR errors:
    
    #{Enum.join(citations, "\n")}
    
    Common issues:
    - Year errors (2O23 → 2023)
    - Author name misspellings
    - Journal abbreviation errors
    
    List corrections needed.
    """
    
    OllamaClient.generate(prompt, model: model)
  end
end

Mathematical Content Detection

defmodule Codexis.QualityChecker.MathDetector do
  @moduledoc """
  Specialized detection for mathematical content
  """
  
  alias Codexis.LLM.OllamaClient
  
  @math_correction_schema %{
    corrections: %{
      type: "array",
      items: %{
        type: "object",
        properties: %{
          original: %{type: "string"},
          corrected: %{type: "string"},
          type: %{type: "string", enum: ["symbol", "operator", "delimiter", "script"]},
          reason: %{type: "string"},
          confidence: %{type: "number"},
          position: %{type: "integer"}
        }
      }
    }
  }
  
  def check_equation(equation_content, opts \\ []) do
    model = Keyword.get(opts, :model, "deepseek-r1:8b")
    
    prompt = build_math_prompt(equation_content)
    
    case OllamaClient.generate_structured(prompt, @math_correction_schema, model: model) do
      {:ok, result} -> process_math_corrections(result)
      {:error, reason} -> {:error, reason}
    end
  end
  
  defp build_math_prompt(content) do
    """
    Analyze this LaTeX mathematical expression for OCR errors:
    
    #{content}
    
    Focus on:
    1. Greek letter substitutions (α→a, β→b, θ→0)
    2. Operator errors (∑→Σ, ∫→f, ·→.)
    3. Delimiter matching (mismatched brackets/parentheses)
    4. Subscript/superscript placement
    5. Common math symbol corruptions
    
    For each error, identify:
    - The original incorrect symbol/text
    - The corrected version
    - The type of error (symbol, operator, delimiter, script)
    - Why it's likely an error
    - Confidence level (0-1)
    - Character position
    """
  end
  
  defp process_math_corrections(result) do
    result["corrections"]
    |> Enum.map(fn correction ->
      # Add additional validation for math corrections
      if valid_math_correction?(correction) do
        correction
      else
        Map.put(correction, "confidence", correction["confidence"] * 0.7)
      end
    end)
  end
  
  defp valid_math_correction?(correction) do
    # Basic validation for mathematical corrections
    case correction["type"] do
      "symbol" -> valid_symbol_correction?(correction)
      "operator" -> valid_operator_correction?(correction)
      "delimiter" -> valid_delimiter_correction?(correction)
      "script" -> valid_script_correction?(correction)
      _ -> true
    end
  end
  
  defp valid_symbol_correction?(correction) do
    # Check if the correction involves valid mathematical symbols
    valid_symbols = ~w(alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega)
    
    orig = String.downcase(correction["original"])
    corr = String.downcase(correction["corrected"])
    
    Enum.any?(valid_symbols, fn symbol ->
      String.contains?(corr, symbol) || String.contains?(orig, symbol)
    end)
  end
  
  defp valid_operator_correction?(_correction), do: true
  defp valid_delimiter_correction?(_correction), do: true
  defp valid_script_correction?(_correction), do: true
end

LaTeX Command Validation

defmodule Codexis.QualityChecker.CommandValidator do
  @moduledoc """
  Validates and corrects LaTeX commands
  """
  
  alias Codexis.LLM.OllamaClient
  
  @common_commands ~w(
    section subsection subsubsection paragraph
    chapter part appendix
    begin end
    cite citep citet ref label
    includegraphics caption
    textbf textit emph
    frac sqrt sum prod int
    alpha beta gamma delta
    usepackage documentclass
  )
  
  def validate_commands(commands, model \\ "qwen3.0:8b") do
    # First pass: check against known commands
    known_corrections = check_known_commands(commands)
    
    # Second pass: use LLM for uncertain cases
    uncertain_commands = filter_uncertain_commands(commands, known_corrections)
    
    if Enum.empty?(uncertain_commands) do
      known_corrections
    else
      llm_corrections = check_with_llm(uncertain_commands, model)
      merge_corrections(known_corrections, llm_corrections)
    end
  end
  
  defp check_known_commands(commands) do
    commands
    |> Enum.map(fn command ->
      if correction = find_known_correction(command.command) do
        %{
          original: command.full,
          corrected: String.replace(command.full, command.command, correction),
          reason: "Common LaTeX command misspelling",
          confidence: 0.95,
          type: :command
        }
      else
        nil
      end
    end)
    |> Enum.filter(& &1)
  end
  
  defp find_known_correction(command) do
    command_lower = String.downcase(command)
    
    # Find the closest matching command
    @common_commands
    |> Enum.map(fn known ->
      {known, String.jaro_distance(command_lower, known)}
    end)
    |> Enum.filter(fn {_, distance} -> distance > 0.8 end)
    |> Enum.max_by(fn {_, distance} -> distance end, fn -> nil end)
    |> case do
      nil -> nil
      {correct_command, _} -> correct_command
    end
  end
  
  defp filter_uncertain_commands(commands, known_corrections) do
    corrected_commands = Enum.map(known_corrections, & &1.original)
    
    Enum.filter(commands, fn command ->
      not Enum.member?(corrected_commands, command.full)
    end)
  end
  
  defp check_with_llm(commands, model) do
    prompt = """
    Check these LaTeX commands for OCR errors:
    
    #{Enum.map(commands, & &1.full) |> Enum.join("\n")}
    
    Common issues:
    - Misspelled command names
    - Wrong delimiters (parentheses instead of braces)
    - Invalid syntax
    
    For each error, provide the corrected command.
    """
    
    case OllamaClient.generate(prompt, model: model) do
      {:ok, response} -> parse_llm_corrections(response, commands)
      {:error, _} -> []
    end
  end
  
  defp parse_llm_corrections(response, _commands) do
    # Parse the LLM response to extract corrections
    # This is a simplified version - real implementation would be more robust
    []
  end
  
  defp merge_corrections(known, llm) do
    known ++ llm
  end
end

Bibliography and Citation Detection

defmodule Codexis.QualityChecker.BibliographyDetector do
  @moduledoc """
  Detects OCR errors in bibliographic content
  """
  
  alias Codexis.LLM.OllamaClient
  
  def check_bibliography_entry(entry, model \\ "qwen3.0:8b") do
    prompt = """
    Check this bibliography entry for OCR errors:
    
    #{entry}
    
    Common issues:
    1. Author names with character substitutions
    2. Year errors (O instead of 0, l instead of 1)
    3. Journal name corruptions
    4. DOI/URL errors
    5. Page number issues
    
    Provide corrections for any errors found.
    """
    
    OllamaClient.generate(prompt, model: model)
  end
  
  def check_citation_consistency(citations, bibliography, model \\ "qwen3.0:8b") do
    prompt = """
    Check citation consistency between these citations and bibliography entries:
    
    Citations found in text:
    #{Enum.join(citations, "\n")}
    
    Bibliography entries:
    #{Enum.join(bibliography, "\n")}
    
    Look for:
    1. Citations without matching bibliography entries
    2. Bibliography entries that are never cited
    3. Inconsistent citation keys (possible OCR errors)
    
    Report any inconsistencies or errors.
    """
    
    OllamaClient.generate(prompt, model: model)
  end
  
  def validate_dois(dois, model \\ "qwen3.0:8b") do
    prompt = """
    Check these DOIs for OCR errors:
    
    #{Enum.join(dois, "\n")}
    
    DOI format should be: 10.xxxx/yyyy
    
    Common OCR errors in DOIs:
    - O/0 confusion
    - l/1 confusion
    - Missing or extra dots/slashes
    
    Correct any malformed DOIs.
    """
    
    OllamaClient.generate(prompt, model: model)
  end
end

Code Block Detection

defmodule Codexis.QualityChecker.CodeDetector do
  @moduledoc """
  Detects OCR errors in code blocks and listings
  """
  
  alias Codexis.LLM.OllamaClient
  
  def check_code_block(code, language, model \\ "qwen3.0:8b") do
    prompt = """
    Check this #{language} code block for OCR errors:
    
    ```#{language}
    #{code}
    ```
    
    Common OCR errors in code:
    1. Similar character confusion (0/O, 1/l, etc.)
    2. Bracket/parenthesis errors
    3. Operator errors (==/>-, !=/>)
    4. Keyword misspellings
    5. Indentation issues
    
    Provide corrections for any errors that would prevent compilation/execution.
    """
    
    case OllamaClient.generate(prompt, model: model) do
      {:ok, response} -> parse_code_corrections(response)
      {:error, reason} -> {:error, reason}
    end
  end
  
  defp parse_code_corrections(response) do
    # Extract and parse code corrections from LLM response
    %{
      corrections: [],
      fixed_code: response
    }
  end
  
  def validate_syntax(code, language) do
    # Language-specific syntax validation
    case language do
      "python" -> validate_python_syntax(code)
      "elixir" -> validate_elixir_syntax(code)
      "latex" -> validate_latex_syntax(code)
      _ -> {:ok, :unknown_language}
    end
  end
  
  defp validate_python_syntax(code) do
    # Basic Python syntax validation
    {:ok, :valid}
  end
  
  defp validate_elixir_syntax(code) do
    # Use Code.string_to_quoted for validation
    case Code.string_to_quoted(code) do
      {:ok, _} -> {:ok, :valid}
      {:error, {line, error, _}} -> {:error, "Line #{line}: #{error}"}
    end
  end
  
  defp validate_latex_syntax(code) do
    # Basic LaTeX syntax validation
    {:ok, :valid}
  end
end