[Feature] Add vision language model support. (#3042)

2024-03-25 14:16:30 -07:00
parent f408d05c52
commit 64172a976c
28 changed files with 936 additions and 94 deletions
--- a/tests/worker/test_model_runner.py
+++ b/tests/worker/test_model_runner.py
@ -35,7 +35,7 @@ def test_prepare_prompt(batch_size):
                                               prompt_len - 1)
        selected_token_start_idx += prompt_len
    (input_tokens, input_positions, attn_metadata, return_prompt_lens, _, _, _,
-     _) = (model_runner._prepare_prompt(seq_group_metadata_list))
+     _, _) = (model_runner._prepare_prompt(seq_group_metadata_list))
    assert return_prompt_lens == prompt_lens

    # Verify input metadata is correct for prompts.