Back to Blog
How Much Quality Loss Is Acceptable
Manufacturing has tolerances. A part can be 0.1mm off in a washing machine but must be perfect in a pacemaker. The same deviation means nothing in one context and everything in another.
Model quality loss works identically. The same percentage drop that's invisible in one task breaks another completely. Setting the right threshold requires understanding what failure looks like.
The Threshold Framework
def quality_threshold_by_task():
"""
How much degradation is acceptable?
"""
return {
"classification": {
"acceptable_loss": "3-5%",
"reasoning": "Binary output, errors are clear",
"failure_mode": "Mislabeled items",
"user_impact": "Usually caught by other systems",
},
"summarization": {
"acceptable_loss": "2-4%",
"reasoning": "Content matters, exact words less so",
"failure_mode": "Less crisp summaries",
"user_impact": "Mild quality degradation",
},
"chat_conversation": {
"acceptable_loss": "2-3%",
"reasoning": "Variety masks small quality changes",
"failure_mode": "Slightly less coherent responses",
"user_impact": "Barely noticeable usually",
},
"code_generation": {
"acceptable_loss": "0.5-1%",
"reasoning": "Syntax errors break everything",
"failure_mode": "Broken code, subtle bugs",
"user_impact": "Potentially severe",
},
"math_reasoning": {
"acceptable_loss": "0.5-1%",
"reasoning": "Wrong numbers cascade",
"failure_mode": "Incorrect calculations",
"user_impact": "High if users trust outputs",
},
"legal_medical": {
"acceptable_loss": "< 0.5%",
"reasoning": "Errors have real consequences",
"failure_mode": "Incorrect advice",
"user_impact": "Potentially dangerous",
},
}Measuring Quality Degradation
class QualityMeasurement:
"""
How to measure the actual loss
"""
metrics_by_type = {
"generation_tasks": {
"primary": "Human preference (A/B test)",
"secondary": ["ROUGE", "BERTScore", "GPT-4 eval"],
"automated": "Use LLM-as-judge with calibration",
},
"classification": {
"primary": "Accuracy, F1",
"secondary": ["Precision", "Recall by class"],
"automated": "Standard ML metrics",
},
"code": {
"primary": "Pass@k on test cases",
"secondary": ["Syntax validity rate", "Static analysis"],
"automated": "Execute and check",
},
"extraction": {
"primary": "Exact match rate",
"secondary": ["Partial match", "Field accuracy"],
"automated": "Structured comparison",
},
}
def measure_degradation(self, baseline_scores: dict, optimized_scores: dict) -> dict:
degradation = {}
for metric, baseline in baseline_scores.items():
optimized = optimized_scores[metric]
degradation[metric] = {
"baseline": baseline,
"optimized": optimized,
"absolute_drop": baseline - optimized,
"relative_drop": (baseline - optimized) / baseline * 100,
}
return degradationSetting Your Threshold
def threshold_setting_process():
return {
"step_1": {
"action": "Define failure for your use case",
"questions": [
"What happens when the model is wrong?",
"How do users discover errors?",
"What's the cost of an error?",
],
},
"step_2": {
"action": "Measure baseline quality",
"questions": [
"What's the current error rate?",
"Where do current errors occur?",
"Are users satisfied with current quality?",
],
},
"step_3": {
"action": "Define acceptable degradation",
"questions": [
"Would users notice X% more errors?",
"Would X% more errors cause harm?",
"What's the business impact of X% more errors?",
],
},
"step_4": {
"action": "Set conservative initial threshold",
"recommendation": "Start tighter than you think necessary",
},
"step_5": {
"action": "Monitor and adjust",
"recommendation": "Loosen only with production evidence",
},
}The Cost-Quality Curve
def cost_quality_tradeoff():
"""
Understanding the curve
"""
typical_curve = {
"fp16_baseline": {
"quality": "100%",
"cost": "1.0x",
"note": "Reference point",
},
"int8_weights": {
"quality": "98-99%",
"cost": "0.5x",
"note": "Usually worth it",
},
"int8_weights_kv": {
"quality": "97-98%",
"cost": "0.35x",
"note": "Still usually worth it",
},
"int4_weights": {
"quality": "94-97%",
"cost": "0.25x",
"note": "Task-dependent",
},
"int4_aggressive": {
"quality": "90-95%",
"cost": "0.2x",
"note": "Only for tolerant tasks",
},
}
insight = """
The first 50% cost reduction costs 1-2% quality.
The next 25% costs another 2-3%.
The last 25% costs 5-10%.
Diminishing returns on cost, increasing quality cost.
"""
return typical_curve, insightRed Lines
def quality_red_lines():
"""
Degradation levels that should block deployment
"""
return {
"always_block": [
{
"threshold": "> 5% accuracy drop on core task",
"reason": "Users will notice and complain",
},
{
"threshold": "> 10% increase in error rate",
"reason": "Trust erosion is hard to recover",
},
{
"threshold": "Any increase in safety violations",
"reason": "Non-negotiable",
},
{
"threshold": "Regression on edge cases > 20%",
"reason": "Edge cases are where harm happens",
},
],
"context_dependent": [
{
"threshold": "> 2% drop in code correctness",
"context": "If code generation is core product",
},
{
"threshold": "> 3% drop in factual accuracy",
"context": "If accuracy is primary value prop",
},
],
}Testing Protocol
def quality_testing_protocol():
return [
{
"phase": "Automated benchmarks",
"tests": [
"Standard academic benchmarks",
"Internal task-specific benchmarks",
"Regression test suite",
],
"threshold": "Must pass before human review",
},
{
"phase": "Side-by-side evaluation",
"tests": [
"Human preference test (100+ samples)",
"Expert review on critical cases",
],
"threshold": "Statistical significance of no degradation",
},
{
"phase": "Production shadow",
"tests": [
"Run optimized model on production traffic",
"Compare outputs (not serve to users)",
],
"threshold": "No unexpected divergence patterns",
},
{
"phase": "Gradual rollout",
"tests": [
"1% → 10% → 50% → 100%",
"Monitor user feedback at each stage",
],
"threshold": "No quality complaints increase",
},
]The Decision
def make_the_decision(
measured_degradation: float,
task_type: str,
cost_savings: float,
) -> str:
thresholds = {
"classification": 5,
"summarization": 4,
"chat": 3,
"code": 1,
"math": 1,
"critical": 0.5,
}
acceptable = thresholds.get(task_type, 2)
if measured_degradation <= acceptable:
return f"Accept: {measured_degradation}% degradation is within {acceptable}% threshold"
elif measured_degradation <= acceptable * 1.5:
return f"Review: {measured_degradation}% is borderline, get additional data"
else:
return f"Reject: {measured_degradation}% exceeds {acceptable}% threshold"Quality thresholds aren't universal. They depend on your task, your users, and your cost pressures. Set them thoughtfully, test rigorously, and adjust based on real-world feedback.