Runtime Adaptive Compression: Technical Specification

Status: Ready for implementation
Target Models: Phi-3-mini (MVP), Gemma4-31b (follow-up)
Expected Speedup: 1.70–1.80x with <2% accuracy loss

1. Data Structures

1.1 CliffAtlas

/// Stores singular value ratios (σ₁/σ₂) per layer, enables O(1) lookup.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct CliffAtlas {
    /// Top singular value per layer
    pub sigma_1: Vec<f32>,
    /// Second singular value per layer (used for cliff ratio)
    pub sigma_2: Vec<f32>,
    /// Optional: power-law exponent α (for noise-color classification)
    pub alpha: Option<Vec<f32>>,
    /// Metadata: model name, measurement tokens, etc.
    pub metadata: std::collections::HashMap<String, String>,
}

impl CliffAtlas {
    /// Compute McNally Cliff ratio for a given layer.
    /// Returns σ₁/σ₂; ratio ≥ 8.0 indicates safe compression zone.
    pub fn cliff_ratio(&self, layer_idx: usize) -> f32 {
        if layer_idx >= self.sigma_1.len() {
            return 0.0; // Out of bounds → treat as no cliff
        }
        self.sigma_1[layer_idx] / self.sigma_2[layer_idx].max(1e-6)
    }

    /// Load from spectral-atlas JSONL output.
    /// Expects one JSON object per line with fields: layer, sigma_1, sigma_2, alpha.
    pub fn from_jsonl(path: &str) -> Result<Self, Box<dyn std::error::Error>> {
        use std::fs::File;
        use std::io::{BufRead, BufReader};

        let file = File::open(path)?;
        let reader = BufReader::new(file);

        let mut sigma_1 = Vec::new();
        let mut sigma_2 = Vec::new();
        let mut alpha = Some(Vec::new());

        for line in reader.lines() {
            let line = line?;
            if line.starts_with('#') {
                continue; // Skip comments
            }
            let row: serde_json::Value = serde_json::from_str(&line)?;
            
            let s1 = row["sigma_1"].as_f64().unwrap_or(0.0) as f32;
            let s2 = row["sigma_2"].as_f64().unwrap_or(1e-6) as f32;
            let a = row["alpha"].as_f64().map(|v| v as f32);

            sigma_1.push(s1);
            sigma_2.push(s2);
            if let Some(ref mut alpha_vec) = alpha {
                alpha_vec.push(a.unwrap_or(0.0));
            }
        }

        let metadata = std::collections::HashMap::new(); // Can add timestamps, model name, etc.

        Ok(CliffAtlas {
            sigma_1,
            sigma_2,
            alpha,
            metadata,
        })
    }

    /// Save to JSON for inspection/versioning.
    pub fn to_json(&self) -> Result<String, Box<dyn std::error::Error>> {
        Ok(serde_json::to_string_pretty(self)?)
    }
}

1.2 CompressionPolicy

/// Selects how to compress each layer based on architecture/cliff info.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CompressionPolicy {
    /// No compression
    None,
    /// Fixed ratio across all layers (e.g., 0.5 = 50% = Variant B)
    Fixed { ratio: f32 },
    /// Adaptive per McNally Cliff: compress only where σ₁/σ₂ ≥ 8
    AdaptiveMcNally {
        /// Threshold for cliff (typically 8.0)
        cliff_threshold: f32,
        /// Compression ratio in "valley" (cliff ≥ threshold)
        valley_ratio: f32,
        /// Compression ratio in "transition" zone (5 ≤ cliff < threshold)
        transition_ratio: f32,
    },
    /// Smooth gradient: early layers preserved, late layers compressed
    AdaptiveGradient,
}

impl CompressionPolicy {
    /// Get the intermediate dimension for a given layer.
    /// `original_m`: typically 8192 (Phi-3) or 21504 (Gemma4)
    pub fn get_intermediate_dim(
        &self,
        layer_idx: usize,
        original_m: usize,
        cliff_atlas: Option<&CliffAtlas>,
    ) -> usize {
        match self {
            CompressionPolicy::None => original_m,

            CompressionPolicy::Fixed { ratio } => {
                ((original_m as f32) * ratio).ceil() as usize
            }

            CompressionPolicy::AdaptiveMcNally {
                cliff_threshold,
                valley_ratio,
                transition_ratio,
            } => {
                let cliff = cliff_atlas.map(|a| a.cliff_ratio(layer_idx)).unwrap_or(0.0);

                if cliff >= *cliff_threshold {
                    // Gnostic Valley: safe for aggressive compression
                    ((original_m as f32) * valley_ratio).ceil() as usize
                } else if cliff >= 5.0 {
                    // Transition zone: moderate compression
                    ((original_m as f32) * transition_ratio).ceil() as usize
                } else {
                    // Fragile/white-noise: preserve
                    original_m
                }
            }

            CompressionPolicy::AdaptiveGradient => {
                // Smooth per-layer gradient: early → preserve, late → compress
                let num_layers = 32; // TODO: parameterize or infer
                let ratio = if layer_idx < 8 {
                    0.80 // 80% capacity
                } else if layer_idx < 16 {
                    0.70 // 70%
                } else if layer_idx < 24 {
                    0.60 // 60%
                } else {
                    0.50 // 50%
                };
                ((original_m as f32) * ratio).ceil() as usize
            }
        }
    }
}

1.3 CompressionConfig

/// Wraps policy and atlas for use in inference pipelines.
pub struct CompressionConfig {
    pub policy: CompressionPolicy,
    pub cliff_atlas: Option<CliffAtlas>,
}

impl CompressionConfig {
    pub fn none() -> Self {
        CompressionConfig {
            policy: CompressionPolicy::None,
            cliff_atlas: None,
        }
    }

    pub fn fixed(ratio: f32) -> Self {
        CompressionConfig {
            policy: CompressionPolicy::Fixed { ratio },
            cliff_atlas: None,
        }
    }

    pub fn adaptive_mcnally(
        cliff_atlas: CliffAtlas,
        cliff_threshold: f32,
        valley_ratio: f32,
        transition_ratio: f32,
    ) -> Self {
        CompressionConfig {
            policy: CompressionPolicy::AdaptiveMcNally {
                cliff_threshold,
                valley_ratio,
                transition_ratio,
            },
            cliff_atlas: Some(cliff_atlas),
        }
    }

    pub fn adaptive_gradient() -> Self {
        CompressionConfig {
            policy: CompressionPolicy::AdaptiveGradient,
            cliff_atlas: None,
        }
    }
}

2. Integration Points

2.1 Phi3Pipeline

Modify the Phi3Pipeline struct to accept and use compression config:

pub struct Phi3Pipeline {
    // ... existing fields ...
    
    /// Optional: adaptive compression policy
    pub compression_config: Option<CompressionConfig>,
}

impl Phi3Pipeline {
    pub fn new(/* ... existing args ... */, compression_config: Option<CompressionConfig>) -> Self {
        Phi3Pipeline {
            // ... existing init ...
            compression_config,
        }
    }

    /// Forward one layer's FFN, respecting compression policy.
    fn forward_ffn(
        &mut self,
        layer_idx: usize,
        x: &[f32],  // residual, shape [batch × hidden]
    ) -> Result<Vec<f32>, String> {
        let m = if let Some(ref config) = self.compression_config {
            config.policy.get_intermediate_dim(layer_idx, 8192, config.cliff_atlas.as_ref())
        } else {
            8192 // No compression
        };

        if m == 8192 {
            // Standard FFN
            self.forward_ffn_standard(layer_idx, x)
        } else {
            // Low-rank FFN
            let mut ffn = LowRankFFNLayer::new(
                layer_idx,
                3072,    // hidden_dim for Phi-3
                8192,    // original_intermediate
                m,       // compressed intermediate
            );
            ffn.load_from_full_weights(
                &self.weights.gate[layer_idx],
                &self.weights.up[layer_idx],
                &self.weights.down[layer_idx],
            )?;
            ffn.forward(x)
        }
    }
}

2.2 Alternative: Lazy Loading with Caching

For production where weights are cached (not reloaded per layer):

pub struct Phi3PipelineWithCache {
    // ... existing fields ...
    compression_config: Option<CompressionConfig>,
    
    /// Cache lowrank FFNs to avoid repeated weight slicing
    lowrank_cache: std::collections::HashMap<(usize, usize), Arc<LowRankFFNLayer>>,
}

impl Phi3PipelineWithCache {
    fn forward_ffn(&mut self, layer_idx: usize, x: &[f32]) -> Result<Vec<f32>, String> {
        let m = if let Some(ref config) = self.compression_config {
            config.policy.get_intermediate_dim(layer_idx, 8192, config.cliff_atlas.as_ref())
        } else {
            8192
        };

        let cache_key = (layer_idx, m);
        if let Some(ffn) = self.lowrank_cache.get(&cache_key) {
            return ffn.forward(x);
        }

        // Load and cache
        let mut ffn = LowRankFFNLayer::new(layer_idx, 3072, 8192, m);
        ffn.load_from_full_weights(
            &self.weights.gate[layer_idx],
            &self.weights.up[layer_idx],
            &self.weights.down[layer_idx],
        )?;

        let result = ffn.forward(x)?;
        self.lowrank_cache.insert(cache_key, Arc::new(ffn));
        Ok(result)
    }
}

3. Measurement & Calibration

3.1 Measuring Cliff Atlas

Run spectral-atlas on each model to produce the atlas:

# For Phi-3-mini
cargo run --bin spectral-atlas --release -- \
  --knot /path/to/phi3-mini.knot \
  --token-range 0..256 \
  --out phi3-cliff-atlas.jsonl

# For Gemma4
cargo run --bin spectral-atlas --release -- \
  --knot /path/to/gemma4-31b-it.knot \
  --token-range 0..512 \
  --out gemma4-cliff-atlas.jsonl

3.2 Verifying Cliff Threshold

Use diagnose-variant-c-cliff.rs as reference. Expected pattern:

Layer   Cliff Ratio   Zone
0–10    1.2–2.0      White noise (no cliff)
11–20   5–40          Brown (Gnostic Valley, sharp cliff)
21–31   1.3–2.0      Fading (late refinement)

Threshold decision:

σ₁/σ₂ ≥ 8.0: Formal McNally Cliff (safe for 50% compression)
σ₁/σ₂ ∈ [5, 8): Transition zone (recommend 65% compression)
σ₁/σ₂ < 5.0: Fragile (preserve full capacity)

4. Benchmarking

4.1 Speedup Benchmark

// bench/adaptive_compression_throughput.rs
use criterion::*;

fn bench_policies(c: &mut Criterion) {
    let knot = load_phi3_mini();
    let prompts = load_mmlu_prompts(100); // Representative sample

    c.bench_function("compression_none", |b| {
        b.iter(|| {
            let mut pipeline = Phi3Pipeline::new(knot.clone(), None);
            for prompt in &prompts {
                let _ = pipeline.generate(prompt, 128);
            }
        })
    });

    c.bench_function("compression_fixed_50", |b| {
        b.iter(|| {
            let config = CompressionConfig::fixed(0.5);
            let mut pipeline = Phi3Pipeline::new(knot.clone(), Some(config));
            for prompt in &prompts {
                let _ = pipeline.generate(prompt, 128);
            }
        })
    });

    c.bench_function("compression_adaptive_mcnally", |b| {
        b.iter(|| {
            let atlas = CliffAtlas::from_jsonl("phi3-cliff-atlas.jsonl").unwrap();
            let config = CompressionConfig::adaptive_mcnally(atlas, 8.0, 0.5, 0.65);
            let mut pipeline = Phi3Pipeline::new(knot.clone(), Some(config));
            for prompt in &prompts {
                let _ = pipeline.generate(prompt, 128);
            }
        })
    });
}

Expected output:

compression_none              time:   [2.341 s 2.356 s 2.371 s]
compression_fixed_50          time:   [1.189 s 1.202 s 1.215 s]  (1.97x)
compression_adaptive_mcnally  time:   [1.382 s 1.401 s 1.420 s]  (1.70x)

4.2 Accuracy Benchmark (MMLU 5-shot)

// bin/mmlu_eval_adaptive.rs
use phi3_llm::*;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut results = Vec::new();

    for policy_name in ["none", "fixed_50", "adaptive_mcnally"] {
        let config = match policy_name {
            "none" => None,
            "fixed_50" => Some(CompressionConfig::fixed(0.5)),
            "adaptive_mcnally" => {
                let atlas = CliffAtlas::from_jsonl("phi3-cliff-atlas.jsonl")?;
                Some(CompressionConfig::adaptive_mcnally(atlas, 8.0, 0.5, 0.65))
            }
            _ => panic!("unknown policy"),
        };

        let mut pipeline = Phi3Pipeline::new(knot, config);
        let mut correct = 0;
        let mut total = 0;

        for question in load_mmlu_questions(1000) {
            let answer = pipeline.generate_answer(&question, 5)?;
            if answer == question.correct_answer {
                correct += 1;
            }
            total += 1;
        }

        let accuracy = correct as f32 / total as f32;
        results.push((policy_name, accuracy));
        println!("{}: {:.1}%", policy_name, accuracy * 100.0);
    }

    // Analyze
    let baseline = results[0].1;
    for (policy, acc) in &results[1..] {
        let loss = (baseline - acc) * 100.0;
        println!("  Δ accuracy vs. baseline: {:.2}%", loss);
    }

    Ok(())
}

Expected output:

none: 72.5%
fixed_50: 71.1%  (Δ -1.4%)
adaptive_mcnally: 71.3%  (Δ -1.2%)

5. Testing

5.1 Unit Tests

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_cliff_ratio_lookup() {
        let mut atlas = CliffAtlas {
            sigma_1: vec![10.0, 8.0, 6.0],
            sigma_2: vec![1.0, 1.0, 1.0],
            alpha: None,
            metadata: Default::default(),
        };

        assert_eq!(atlas.cliff_ratio(0), 10.0);
        assert_eq!(atlas.cliff_ratio(1), 8.0);
        assert_eq!(atlas.cliff_ratio(2), 6.0);
        assert_eq!(atlas.cliff_ratio(999), 0.0); // Out of bounds
    }

    #[test]
    fn test_policy_fixed() {
        let policy = CompressionPolicy::Fixed { ratio: 0.5 };
        assert_eq!(policy.get_intermediate_dim(0, 8192, None), 4096);
    }

    #[test]
    fn test_policy_adaptive_mcnally() {
        let mut atlas = CliffAtlas {
            sigma_1: vec![40.0, 2.0, 40.0],
            sigma_2: vec![1.0, 1.0, 1.0],
            alpha: None,
            metadata: Default::default(),
        };

        let policy = CompressionPolicy::AdaptiveMcNally {
            cliff_threshold: 8.0,
            valley_ratio: 0.5,
            transition_ratio: 0.65,
        };

        // Layer 0: cliff=40 ≥ 8 → 50% compression
        assert_eq!(policy.get_intermediate_dim(0, 8192, Some(&atlas)), 4096);

        // Layer 1: cliff=2 < 5 → no compression
        assert_eq!(policy.get_intermediate_dim(1, 8192, Some(&atlas)), 8192);

        // Layer 2: cliff=40 ≥ 8 → 50% compression
        assert_eq!(policy.get_intermediate_dim(2, 8192, Some(&atlas)), 4096);
    }

    #[test]
    fn test_policy_gradient() {
        let policy = CompressionPolicy::AdaptiveGradient;

        assert_eq!(policy.get_intermediate_dim(0, 8192, None), 6554);  // 80%
        assert_eq!(policy.get_intermediate_dim(8, 8192, None), 5734);  // 70%
        assert_eq!(policy.get_intermediate_dim(16, 8192, None), 4915); // 60%
        assert_eq!(policy.get_intermediate_dim(24, 8192, None), 4096); // 50%
    }
}

5.2 Integration Tests

#[test]
fn test_adaptive_forward_pass() {
    let knot = load_test_weights();
    let atlas = CliffAtlas::from_jsonl("test_atlas.jsonl").unwrap();
    let config = CompressionConfig::adaptive_mcnally(atlas, 8.0, 0.5, 0.65);

    let mut pipeline = Phi3Pipeline::new(knot, Some(config));
    let prompt = "Hello, how are you?";
    let output = pipeline.generate(prompt, 32).expect("generate failed");

    assert!(!output.is_empty());
    assert!(output.len() < 1000); // Sanity check
}

6. Production Checklist

CliffAtlas struct implemented and tested
CompressionPolicy enum with three variants
Integration into Phi3Pipeline.forward_ffn()
Spectral-atlas output for Phi-3-mini
Speedup benchmark (expected: 1.70x)
MMLU 1k-sample evaluation (expected: <2% loss)
Documentation updated
Phi-3 atlas committed to repo
Optional: Head-level cliff analysis (Angle 5 extension)
Optional: Gemma4 port (requires gemma4 atlas)

7. Future Extensions

7.1 Head-Level Compression (Angle 5 Orthogonal)

Extend to per-head cliff ratios:

pub struct HeadCliffAtlas {
    // [layer][head] cliff ratios
    cliff_by_head: Vec<Vec<f32>>,
}

impl HeadCliffAtlas {
    pub fn should_compress_head(&self, layer: usize, head: usize, threshold: f32) -> bool {
        self.cliff_by_head[layer][head] >= threshold
    }
}

// In attention forward:
for head in 0..num_heads {
    let compress_v = head_cliff_atlas.should_compress_head(layer, head, 8.0);
    if compress_v {
        let v_m = (v_stride as f32 * 0.75).ceil() as usize; // 75% capacity
        // ... proceed with compressed V
    }
}

7.2 Dynamic Threshold Tuning

Expose threshold as operator-adjustable knob:

pub struct AdaptiveCompressionConfig {
    pub cliff_threshold: f32,      // Default 8.0
    pub valley_ratio: f32,         // Default 0.5 (50%)
    pub transition_ratio: f32,     // Default 0.65 (65%)
}

impl AdaptiveCompressionConfig {
    pub fn set_speedup_target(&mut self, speedup: f32) {
        // Calibration curve: speedup vs. threshold
        // For Phi-3: 1.0x (threshold=inf), 1.5x (threshold≈20), 1.7x (threshold=8.0), 2.0x (threshold=0)
        self.cliff_threshold = match speedup {
            s if s <= 1.0 => f32::INFINITY,
            s if s < 1.5 => 20.0,
            s if s < 1.7 => 8.0,
            _ => 0.0,
        };
    }
}

References

docs/ADAPTIVE_COMPRESSION_MCNALLY_CLIFF_POC.md — Motivation, cost analysis, validation plan
src/model_phi3_lowrank.rs — LowRankFFNLayer implementation
src/bin/spectral-atlas.rs — Cliff ratio measurement
src/bin/diagnose-variant-c-cliff.rs — Variant C failure analysis
PHI3_LOWRANK_ANALYSIS.md — Baseline Variant A/B/C results

forkjoin-ai/gnosis