manually manage stream

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
2025-05-20 23:55:31 +00:00
parent 020269c4c5
commit ffb740ae95
4 changed files with 41 additions and 22 deletions
--- a/examples/offline_inference/basic/basic.py
+++ b/examples/offline_inference/basic/basic.py
@ -34,8 +34,9 @@ def main():
              enforce_eager=False,
              compilation_config=2,
              enable_microbatching=True,
+              enable_expert_parallel=True,
              trust_remote_code=True,
-              tensor_parallel_size=4,
+              tensor_parallel_size=2,
              max_model_len=1024,
              #load_format="dummy",
    )