[Meta] Llama4 EAGLE Support (#20591)

Signed-off-by: qizixi <qizixi@meta.com> Co-authored-by: qizixi <qizixi@meta.com>
2025-07-15 21:14:15 -07:00
parent 1eb2b9c102
commit c11013db8b
6 changed files with 255 additions and 15 deletions
--- a/examples/offline_inference/spec_decode.py
+++ b/examples/offline_inference/spec_decode.py
@ -84,6 +84,7 @@ def main():
        gpu_memory_utilization=0.8,
        speculative_config=speculative_config,
        disable_log_stats=False,
+        max_model_len=16384,
    )

    sampling_params = SamplingParams(temperature=args.temp, max_tokens=args.output_len)