Default to generation_config from model (#12622)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

tests/entrypoints/openai/correctness/test_lmeval.py

@@ -20,7 +20,7 @@ NUM_CONCURRENT = 500
 TASK = "gsm8k"
 FILTER = "exact_match,strict-match"
 RTOL = 0.03
-EXPECTED_VALUE = 0.58
+EXPECTED_VALUE = 0.54
 DEFAULT_ARGS = ["--max-model-len", "4096", "--disable-log-requests"]
 MORE_ARGS_LIST = [
     [],  # Default

tests/entrypoints/openai/test_serving_chat.py

@@ -38,6 +38,7 @@ class MockModelConfig:
     diff_sampling_param: Optional[dict] = None
     allowed_local_media_path: str = ""
     encoder_config = None
+    generation_config: str = "auto"
 
     def get_diff_sampling_param(self):
         return self.diff_sampling_param or {}