Mapping Quality Against Cost

Investment portfolios have an efficient frontier: the set of portfolios that maximize return for a given risk level. Points below the frontier are inefficient. You could get more return for the same risk, or the same return for less risk. Smart investors only choose points on the frontier.

LLM configurations have the same structure. Each configuration (model, parameters, optimizations) has a quality level and a cost. The efficient frontier contains configurations where you can't improve quality without increasing cost or reduce cost without sacrificing quality. Everything else is suboptimal.

The Quality-Cost Frontier

def quality_cost_frontier():
    """
    Mapping configurations to their quality-cost position
    """
    configurations = [
        {"name": "Opus full", "cost": 15.00, "quality": 0.95},
        {"name": "Sonnet full", "cost": 3.00, "quality": 0.90},
        {"name": "Haiku full", "cost": 0.25, "quality": 0.80},
        {"name": "Sonnet quantized", "cost": 2.00, "quality": 0.88},
        {"name": "Opus with routing", "cost": 5.00, "quality": 0.93},
        {"name": "Cached + Sonnet", "cost": 1.50, "quality": 0.89},
    ]

    # Find Pareto frontier
    frontier = []
    for config in configurations:
        is_dominated = False
        for other in configurations:
            # Another config has same/better quality at lower cost
            if (other["quality"] >= config["quality"] and
                other["cost"] < config["cost"]):
                is_dominated = True
                break
        if not is_dominated:
            frontier.append(config)

    return {
        "all_configurations": configurations,
        "pareto_frontier": frontier,
        "inefficient": [c for c in configurations if c not in frontier],
    }

Measuring Quality

class QualityMeasurement:
    """
    Quantify quality for comparison
    """

    def __init__(self):
        self.eval_suite = [
            ("accuracy", 0.4),     # Weight
            ("relevance", 0.3),
            ("coherence", 0.2),
            ("safety", 0.1),
        ]

    def measure(self, model_config: dict, test_set: list) -> float:
        """
        Composite quality score from eval suite
        """
        scores = {}

        for eval_name, weight in self.eval_suite:
            evaluator = self.get_evaluator(eval_name)
            scores[eval_name] = evaluator.run(model_config, test_set)

        # Weighted average
        quality = sum(
            scores[name] * weight
            for name, weight in self.eval_suite
        )

        return quality

    def breakdown(self, model_config: dict, test_set: list) -> dict:
        """Detailed breakdown by dimension"""
        return {
            name: self.get_evaluator(name).run(model_config, test_set)
            for name, _ in self.eval_suite
        }

Measuring Cost

class CostMeasurement:
    """
    Comprehensive cost calculation
    """

    def __init__(self, pricing: dict):
        self.input_price = pricing["input_per_million"]
        self.output_price = pricing["output_per_million"]
        self.compute_price = pricing.get("compute_per_hour", 0)

    def cost_per_request(self, request_stats: dict) -> float:
        """Calculate cost of a single request"""
        input_tokens = request_stats["input_tokens"]
        output_tokens = request_stats["output_tokens"]

        input_cost = (input_tokens / 1_000_000) * self.input_price
        output_cost = (output_tokens / 1_000_000) * self.output_price

        return input_cost + output_cost

    def monthly_projection(self, daily_requests: int, avg_request: dict) -> dict:
        """Project monthly costs"""
        cost_per = self.cost_per_request(avg_request)
        daily_cost = cost_per * daily_requests
        monthly_cost = daily_cost * 30

        return {
            "per_request": cost_per,
            "daily": daily_cost,
            "monthly": monthly_cost,
            "annual": monthly_cost * 12,
        }

Building the Frontier Map

class FrontierMapper:
    """
    Map configurations to quality-cost space
    """

    def __init__(self):
        self.quality_measurer = QualityMeasurement()
        self.cost_measurer = CostMeasurement()
        self.test_set = self.load_representative_test_set()

    def map_configuration(self, config: dict) -> dict:
        """Place a configuration on the quality-cost map"""
        quality = self.quality_measurer.measure(config, self.test_set)
        cost = self.cost_measurer.cost_per_request(
            self.run_sample_requests(config)
        )

        return {
            "config": config,
            "quality": quality,
            "cost": cost,
            "quality_per_dollar": quality / cost if cost > 0 else float("inf"),
        }

    def map_all_configurations(self, configs: list) -> list:
        """Map multiple configurations"""
        return [self.map_configuration(c) for c in configs]

    def find_frontier(self, mapped_configs: list) -> list:
        """Identify Pareto optimal configurations"""
        frontier = []

        for config in mapped_configs:
            dominated = False
            for other in mapped_configs:
                if (other["quality"] >= config["quality"] and
                    other["cost"] < config["cost"] and
                    (other["quality"] > config["quality"] or
                     other["cost"] < config["cost"])):
                    dominated = True
                    break

            if not dominated:
                frontier.append(config)

        # Sort by cost
        frontier.sort(key=lambda x: x["cost"])
        return frontier

Choosing Your Spot

def choosing_configuration():
    return {
        "cost_constrained": {
            "situation": "Fixed budget, maximize quality",
            "approach": """
                frontier = find_frontier(configs)
                affordable = [c for c in frontier if c.cost <= budget]
                best = max(affordable, key=lambda c: c.quality)
            """,
            "example": "Budget is $1K/month, pick highest quality under that",
        },

        "quality_constrained": {
            "situation": "Must meet quality bar, minimize cost",
            "approach": """
                frontier = find_frontier(configs)
                acceptable = [c for c in frontier if c.quality >= min_quality]
                cheapest = min(acceptable, key=lambda c: c.cost)
            """,
            "example": "Quality must be 90%+, find cheapest option that achieves it",
        },

        "balanced": {
            "situation": "Optimize quality per dollar",
            "approach": """
                frontier = find_frontier(configs)
                best_value = max(frontier, key=lambda c: c.quality / c.cost)
            """,
            "example": "Find the configuration with best quality per dollar spent",
        },

        "tier_based": {
            "situation": "Different requirements for different users",
            "approach": """
                premium_config = max(frontier, key=lambda c: c.quality)
                standard_config = frontier[len(frontier)//2]  # Middle of frontier
                basic_config = min(frontier, key=lambda c: c.cost)
            """,
            "example": "Offer gold/silver/bronze tiers along the frontier",
        },
    }

Visualizing the Tradeoff

def visualization_approach():
    return """
    Quality
    ^
    |         * Opus full (expensive, best)
    |       *
    |     *   Pareto frontier
    |   *
    | *
    |*  Haiku (cheap, acceptable)
    |
    |     x  x  x   (Inefficient configurations)
    |   x
    +------------------------> Cost

    Key insights:
    1. Points on the curve are valid choices
    2. Points below the curve are mistakes
    3. Moving right on the curve buys quality
    4. Your choice depends on your constraints
    """

Dynamic Frontier Selection

class DynamicFrontierSelector:
    """
    Choose configuration based on request characteristics
    """

    def __init__(self, frontier_configs: list):
        self.frontier = frontier_configs
        self.default_config = self.find_balanced()

    def select_for_request(self, request: dict) -> dict:
        """Choose configuration based on request properties"""

        # High-value users get premium config
        if request.get("user_tier") == "enterprise":
            return self.high_quality_config()

        # Cost-sensitive endpoints get economical config
        if request.get("endpoint") == "bulk_process":
            return self.low_cost_config()

        # Simple tasks don't need expensive models
        task_complexity = self.estimate_complexity(request)
        if task_complexity < 0.3:
            return self.low_cost_config()
        elif task_complexity > 0.8:
            return self.high_quality_config()

        return self.default_config

    def high_quality_config(self) -> dict:
        return max(self.frontier, key=lambda c: c["quality"])

    def low_cost_config(self) -> dict:
        return min(self.frontier, key=lambda c: c["cost"])

    def find_balanced(self) -> dict:
        """Configuration with best quality/cost ratio"""
        return max(self.frontier, key=lambda c: c["quality"] / c["cost"])

Shifting the Frontier

def improving_frontier():
    return {
        "optimizations_that_shift_frontier": [
            {
                "technique": "Prompt caching",
                "effect": "Same quality, lower cost",
                "frontier_shift": "Left (cost reduction)",
            },
            {
                "technique": "Better quantization",
                "effect": "Near-same quality, lower cost",
                "frontier_shift": "Left and slightly down",
            },
            {
                "technique": "Fine-tuning for task",
                "effect": "Higher quality, same cost",
                "frontier_shift": "Up (quality improvement)",
            },
            {
                "technique": "Smart routing",
                "effect": "Match quality, reduce average cost",
                "frontier_shift": "Left (cost reduction)",
            },
        ],

        "goal": """
        Each optimization should either:
        1. Move your current point closer to frontier
        2. Shift the frontier itself (better options available)

        If it does neither, it's not worth the complexity.
        """,
    }

Monitoring Frontier Position

def frontier_monitoring():
    return {
        "track_over_time": [
            "Your configuration's quality score",
            "Your configuration's cost",
            "Distance from current frontier",
            "How frontier is shifting (new models, techniques)",
        ],

        "review_triggers": [
            "New model released",
            "New optimization technique available",
            "Your quality dropped",
            "Your costs increased",
            "Quarterly review",
        ],

        "questions_to_ask": [
            "Are we on the current frontier?",
            "Has the frontier shifted since we last checked?",
            "Are there better options for our constraint?",
            "Should we move to a different point on the frontier?",
        ],
    }

The quality-cost frontier makes tradeoffs explicit. Every configuration choice is a position on this map. Know where you are, know where the frontier is, and choose your spot deliberately based on your constraints. Points off the frontier are money left on the table.