[Misc] Add warning for multimodal data in LLM.beam_search (vllm-project#16241)

Signed-off-by: Alex-Brooks <[email protected]>
Signed-off-by: Yang Wang <[email protected]>

Author: alex-jw-brooks

vllm/entrypoints/llm.py

@@ -536,6 +536,16 @@ def sort_beams_key(x: BeamSearchSequence) -> float:
                                          tokenizer.eos_token_id,
                                          length_penalty)
 
+        # TODO - fix handling of multimodal data for beam search; we pass it
+        # through in the async version on the abstract EngineClient, but not
+        # here.
+        if any("multi_modal_data" in prompt
+               and prompt["multi_modal_data"] is not None
+               for prompt in prompts):
+            logger.warning(
+                "Multimodal data appears to have been provided, but is not"
+                " currently being passed through in LLM.beam_search()!")
+
         tokenizer = self.get_tokenizer()
         # generate 2 * beam_width candidates at each step
         # following the huggingface transformers implementation