LaTeX Content Processor

The LaTeX processor extracts and prepares content from LaTeX manuscripts for scanno detection. It handles the unique challenges of processing academic LaTeX documents.

Core Implementation

defmodule Codexis.QualityChecker.LaTeXProcessor do
  @moduledoc """
  Processes LaTeX content for scanno detection
  """
  
  def extract_checkable_content(latex_content) do
    %{
      text_blocks: extract_text_blocks(latex_content),
      commands: extract_commands(latex_content),
      math_blocks: extract_math_blocks(latex_content),
      references: extract_references(latex_content)
    }
  end
  
  defp extract_text_blocks(content) do
    # Extract paragraphs and text content, excluding commands and math
    content
    |> String.split(~r/\n\s*\n/)
    |> Enum.map(&clean_text_block/1)
    |> Enum.filter(&valid_text_block?/1)
    |> Enum.with_index()
    |> Enum.map(fn {block, index} ->
      %{
        content: block,
        index: index,
        type: :text,
        line_start: calculate_line_number(content, block)
      }
    end)
  end
  
  defp clean_text_block(block) do
    block
    |> String.replace(~r/\\[a-zA-Z]+\{[^}]*\}/, "")  # Remove simple commands
    |> String.replace(~r/\$[^$]+\$/, "[MATH]")       # Replace inline math
    |> String.trim()
  end
  
  defp valid_text_block?(block) do
    String.length(block) > 20 && 
    !String.starts_with?(block, "%") &&
    !String.match?(block, ~r/^\s*\\/)
  end
  
  defp extract_commands(content) do
    Regex.scan(~r/\\([a-zA-Z]+)(?:\[[^\]]*\])?(?:\{[^}]*\})*/, content)
    |> Enum.map(fn [full, command] -> 
      %{
        command: command,
        full: full,
        position: :binary.match(content, full) |> elem(0)
      }
    end)
  end
  
  defp extract_math_blocks(content) do
    # Extract display math
    display_math = Regex.scan(~r/\\\[(.*?)\\\]/s, content)
    |> Enum.map(fn [full, math] -> 
      %{type: :display, content: math, full: full}
    end)
    
    # Extract equation environments
    equation_envs = Regex.scan(~r/\\begin\{equation\}(.*?)\\end\{equation\}/s, content)
    |> Enum.map(fn [full, math] ->
      %{type: :equation, content: math, full: full}
    end)
    
    display_math ++ equation_envs
  end
  
  defp extract_references(content) do
    # Extract bibliography entries and citations
    citations = Regex.scan(~r/\\cite\{([^}]+)\}/, content)
    |> Enum.map(fn [_, refs] -> String.split(refs, ",") end)
    |> List.flatten()
    |> Enum.map(&String.trim/1)
    
    %{citations: citations}
  end
  
  defp calculate_line_number(content, block) do
    # Find the line number where this block starts
    lines = String.split(content, "\n")
    
    Enum.find_index(lines, fn line ->
      String.contains?(line, String.slice(block, 0, 20))
    end) || 0
  end
end

Advanced Processing

defmodule Codexis.QualityChecker.LaTeXAdvanced do
  @moduledoc """
  Advanced LaTeX content processing
  """
  
  def extract_environments(content) do
    # Extract specific LaTeX environments
    environments = [
      "figure", "table", "algorithm", "listing",
      "theorem", "proof", "definition", "lemma"
    ]
    
    Enum.flat_map(environments, fn env ->
      extract_environment(content, env)
    end)
  end
  
  defp extract_environment(content, env_name) do
    pattern = ~r/\\begin\{#{env_name}\}(.*?)\\end\{#{env_name}\}/s
    
    Regex.scan(pattern, content)
    |> Enum.map(fn [full, inner] ->
      %{
        environment: env_name,
        content: inner,
        full: full,
        position: :binary.match(content, full) |> elem(0)
      }
    end)
  end
  
  def extract_sectioning(content) do
    # Extract chapter, section, subsection structure
    commands = [
      "chapter", "section", "subsection", 
      "subsubsection", "paragraph", "subparagraph"
    ]
    
    pattern = ~r/\\(#{Enum.join(commands, "|")})\{([^}]+)\}/
    
    Regex.scan(pattern, content)
    |> Enum.map(fn [full, level, title] ->
      %{
        level: level,
        title: title,
        full: full,
        position: :binary.match(content, full) |> elem(0)
      }
    end)
  end
  
  def extract_custom_commands(content) do
    # Extract custom command definitions
    newcommand_pattern = ~r/\\newcommand\{\\([a-zA-Z]+)\}(?:\[(\d+)\])?\{(.*?)\}/
    
    Regex.scan(newcommand_pattern, content)
    |> Enum.map(fn 
      [full, name, "", definition] ->
        %{
          command: name,
          args: 0,
          definition: definition,
          full: full
        }
      [full, name, args, definition] ->
        %{
          command: name,
          args: String.to_integer(args),
          definition: definition,
          full: full
        }
    end)
  end
end

Context Extraction

defmodule Codexis.QualityChecker.ContextExtractor do
  @moduledoc """
  Extracts contextual information for better scanno detection
  """
  
  def get_document_context(content) do
    %{
      document_class: extract_document_class(content),
      packages: extract_packages(content),
      custom_commands: extract_custom_commands(content),
      bibliography_style: extract_bib_style(content),
      language: detect_language(content)
    }
  end
  
  defp extract_document_class(content) do
    case Regex.run(~r/\\documentclass(?:\[[^\]]*\])?\{([^}]+)\}/, content) do
      [_, class] -> class
      nil -> "article"  # default
    end
  end
  
  defp extract_packages(content) do
    Regex.scan(~r/\\usepackage(?:\[[^\]]*\])?\{([^}]+)\}/, content)
    |> Enum.map(fn [_, package] -> package end)
  end
  
  defp extract_bib_style(content) do
    case Regex.run(~r/\\bibliographystyle\{([^}]+)\}/, content) do
      [_, style] -> style
      nil -> nil
    end
  end
  
  defp detect_language(content) do
    # Simple language detection based on babel package
    case Regex.run(~r/\\usepackage\[([^\]]+)\]\{babel\}/, content) do
      [_, langs] -> String.split(langs, ",") |> List.first()
      nil -> "english"
    end
  end
end

Position Tracking

defmodule Codexis.QualityChecker.PositionTracker do
  @moduledoc """
  Tracks positions in LaTeX documents for accurate correction placement
  """
  
  def create_position_map(content) do
    lines = String.split(content, "\n")
    
    lines
    |> Enum.with_index()
    |> Enum.reduce({0, %{}}, fn {line, line_num}, {char_pos, map} ->
      line_map = %{
        line_num: line_num,
        start_pos: char_pos,
        end_pos: char_pos + String.length(line),
        content: line
      }
      
      new_map = Map.put(map, line_num, line_map)
      {char_pos + String.length(line) + 1, new_map}
    end)
    |> elem(1)
  end
  
  def get_line_from_position(position_map, char_position) do
    position_map
    |> Map.values()
    |> Enum.find(fn %{start_pos: start, end_pos: end_pos} ->
      char_position >= start && char_position <= end_pos
    end)
  end
  
  def get_context_window(content, position, window_size \\ 100) do
    start_pos = max(0, position - window_size)
    end_pos = min(String.length(content), position + window_size)
    
    %{
      before: String.slice(content, start_pos, position - start_pos),
      target: String.slice(content, position, 1),
      after: String.slice(content, position + 1, end_pos - position - 1)
    }
  end
end

Special Content Handlers

defmodule Codexis.QualityChecker.SpecialContent do
  @moduledoc """
  Handles special LaTeX content types
  """
  
  def extract_verbatim_blocks(content) do
    # Extract verbatim and lstlisting blocks
    verbatim_pattern = ~r/\\begin\{verbatim\}(.*?)\\end\{verbatim\}/s
    listing_pattern = ~r/\\begin\{lstlisting\}(.*?)\\end\{lstlisting\}/s
    
    verbatim = extract_with_pattern(content, verbatim_pattern, :verbatim)
    listings = extract_with_pattern(content, listing_pattern, :listing)
    
    verbatim ++ listings
  end
  
  def extract_comments(content) do
    # Extract LaTeX comments for context
    Regex.scan(~r/^%(.*)$/m, content)
    |> Enum.map(fn [full, comment] ->
      %{
        type: :comment,
        content: comment,
        full: full,
        position: :binary.match(content, full) |> elem(0)
      }
    end)
  end
  
  def extract_footnotes(content) do
    # Extract footnotes
    Regex.scan(~r/\\footnote\{([^}]+)\}/, content)
    |> Enum.map(fn [full, footnote] ->
      %{
        type: :footnote,
        content: footnote,
        full: full,
        position: :binary.match(content, full) |> elem(0)
      }
    end)
  end
  
  defp extract_with_pattern(content, pattern, type) do
    Regex.scan(pattern, content)
    |> Enum.map(fn [full, inner] ->
      %{
        type: type,
        content: inner,
        full: full,
        position: :binary.match(content, full) |> elem(0)
      }
    end)
  end
end