Ollama Client Module

The Ollama client module provides a clean interface for interacting with locally-hosted LLMs through the Ollama API.

Core Implementation

defmodule Codexis.LLM.OllamaClient do
  @moduledoc """
  Client for interacting with local Ollama models
  """
  
  @base_url "http://localhost:11434/api"
  @default_model "qwen3.0:8b"
  @timeout 30_000
  
  def generate(prompt, opts \\ []) do
    model = Keyword.get(opts, :model, @default_model)
    temperature = Keyword.get(opts, :temperature, 0.3)
    
    body = %{
      model: model,
      prompt: prompt,
      stream: false,
      options: %{
        temperature: temperature,
        top_p: 0.9,
        repeat_penalty: 1.1
      }
    }
    
    request = Req.new(
      base_url: @base_url,
      timeout: @timeout,
      retry: :transient,
      max_retries: 3
    )
    
    case Req.post(request, url: "/generate", json: body) do
      {:ok, %{status: 200, body: %{"response" => response}}} ->
        {:ok, response}
      {:error, reason} ->
        {:error, reason}
    end
  end
  
  def generate_structured(prompt, schema, opts \\ []) do
    model = Keyword.get(opts, :model, @default_model)
    
    # Format prompt for JSON output
    formatted_prompt = """
    #{prompt}
    
    Respond with a JSON object matching this schema:
    #{Jason.encode!(schema, pretty: true)}
    
    Respond ONLY with valid JSON, no additional text.
    """
    
    case generate(formatted_prompt, model: model, temperature: 0.1) do
      {:ok, response} ->
        parse_json_response(response)
      error ->
        error
    end
  end
  
  defp parse_json_response(response) do
    # Extract JSON from response (handles markdown code blocks)
    json_pattern = ~r/```(?:json)?\s*(.*?)\s*```/s
    
    json_string = case Regex.run(json_pattern, response) do
      [_, json] -> json
      nil -> response
    end
    
    case Jason.decode(json_string) do
      {:ok, parsed} -> {:ok, parsed}
      {:error, _} -> {:error, :invalid_json}
    end
  end
end

Key Features

1. Generate Function: Basic text generation with configurable parameters
2. Structured Output: Support for JSON schema-based responses
3. Error Handling: Robust error handling with retries
4. Configuration: Adjustable temperature, top_p, and repeat penalty

Advanced Options

defmodule Codexis.LLM.OllamaAdvanced do
  @moduledoc """
  Advanced Ollama client features
  """
  
  alias Codexis.LLM.OllamaClient
  
  def generate_with_context(prompt, context, opts \\ []) do
    # Include previous conversation context
    full_prompt = """
    #{format_context(context)}
    
    #{prompt}
    """
    
    OllamaClient.generate(full_prompt, opts)
  end
  
  def stream_generate(prompt, callback, opts \\ []) do
    # Stream responses for real-time output
    model = Keyword.get(opts, :model, "qwen3.0:8b")
    
    body = %{
      model: model,
      prompt: prompt,
      stream: true
    }
    
    # Implementation for streaming responses
  end
  
  defp format_context(context) do
    context
    |> Enum.map(fn %{role: role, content: content} ->
      "#{String.capitalize(role)}: #{content}"
    end)
    |> Enum.join("\n\n")
  end
end

Connection Pooling

defmodule Codexis.LLM.ConnectionPool do
  @moduledoc """
  Connection pool for Ollama requests
  """
  
  use NimblePool
  
  @pool_size 5
  
  def start_link(opts) do
    NimblePool.start_link(
      worker: {__MODULE__, opts},
      pool_size: @pool_size,
      name: __MODULE__
    )
  end
  
  def checkout!(fun) do
    NimblePool.checkout!(
      __MODULE__,
      :checkout,
      fn _from, conn ->
        {result, conn} = fun.(conn)
        {:ok, result, conn}
      end
    )
  end
  
  @impl NimblePool
  def init(_opts) do
    {:ok, nil}
  end
  
  @impl NimblePool
  def handle_checkout(:checkout, from, state) do
    {:ok, state, state}
  end
end

Testing Support

defmodule Codexis.LLM.MockClient do
  @moduledoc """
  Mock Ollama client for testing
  """
  
  def generate(prompt, opts \\ []) do
    # Return predefined responses based on prompt patterns
    cond do
      String.contains?(prompt, "OCR error") ->
        {:ok, mock_ocr_response()}
      String.contains?(prompt, "LaTeX") ->
        {:ok, mock_latex_response()}
      true ->
        {:ok, "Generic mock response"}
    end
  end
  
  defp mock_ocr_response do
    """
    ```json
    {
      "corrections": [
        {
          "original": "tlie",
          "corrected": "the",
          "reason": "Common OCR error",
          "confidence": 0.95,
          "position": 10
        }
      ]
    }
    ```
    """
  end
  
  defp mock_latex_response do
    """
    Found potential error: \\sectiori should be \\section
    """
  end
end