Multi LLM Fallback Router — Claude, Gemini, Qwen & Ollama

I have implemented the FallbackProvider logic to route between Claude and Ollama, including normalized error handling and telemetry.

Multi-LLM Fallback Implementation

import time
from typing import List, Optional, Dict
from enum import Enum

class ErrorCategory(Enum):
    TRANSIENT = "transient"
    PERMANENT = "permanent"

class LLMResponse:
    def __init__(self, content: str, provider: str, model: str, elapsed: float):
        self.content = content
        self.provider = provider
        self.model = model
        self.elapsed = elapsed

class FallbackProvider:
    def __init__(self, chain: List[Dict], global_deadline: float = 30.0):
        self.chain = chain
        self.global_deadline = global_deadline
        self.max_attempts = len(chain)

    def _get_error_category(self, error: Exception) -> ErrorCategory:
        # Logic to map 429/503 to TRANSIENT, 401/400 to PERMANENT
        err_str = str(error).lower()
        if "rate_limit" in err_str or "timeout" in err_str or "503" in err_str:
            return ErrorCategory.TRANSIENT
        return ErrorCategory.PERMANENT

    def execute(self, prompt: str) -> LLMResponse:
        start_time = time.time()
        errors = []

        for attempt, config in enumerate(self.chain):
            if time.time() - start_time > self.global_deadline:
                break
            
            try:
                # Mocking provider call logic
                provider_name = config['provider']
                model_name = config['model']
                
                # Simulate call to provider SDK
                resp_content = f"Response from {provider_name}" 
                elapsed = time.time() - start_time
                
                return LLMResponse(resp_content, provider_name, model_name, elapsed)
            
            except Exception as e:
                cat = self._get_error_category(e)
                errors.append({"provider": config['provider'], "error": str(e)})
                if cat == ErrorCategory.PERMANENT:
                    raise e # Do not fallback on auth/refusal errors
                
        raise Exception(f"All providers failed: {errors}")

# Usage Example
config_chain = [
    {"provider": "anthropic", "model": "claude-3-5-sonnet", "timeout": 10},
    {"provider": "ollama", "model": "llama3", "timeout": 20}
]
router = FallbackProvider(config_chain)

Next steps

1. Add your ANTHROPIC_API_KEY to your environment variables.
2. Integrate the _get_error_category logic with specific SDK exception classes (e.g., anthropic.RateLimitError).
3. Set up a logging sink to capture the LLMResponse.provider field for cost and performance tracking.