max_images are passed on to vllms limit_mm_per_prompt (EleutherAI#2387

)

* max_images are passed on to vllms `limit_mm_per_prompt`

* replace max image placeholders in string

* handle chat_template error

* move `fewshot_random_seed` to global

lm_eval/api/model.py

@@ -431,7 +431,12 @@ def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str
         using_default_template = False
 
         # First, handle the cases when the model has a dict of multiple templates
-        template = self.tokenizer.chat_template or self.tokenizer.default_chat_template
+        try:
+            template = (
+                self.tokenizer.chat_template or self.tokenizer.default_chat_template
+            )
+        except AttributeError:
+            return None
 
         if isinstance(template, dict):
             using_default_dict = self.tokenizer.chat_template is None

lm_eval/evaluator.py

@@ -157,6 +157,9 @@ def simple_evaluate(
         seed_message.append(f"Setting torch manual seed to {torch_random_seed}")
         torch.manual_seed(torch_random_seed)
 
+    if fewshot_random_seed is not None:
+        seed_message.append(f"Setting fewshot manual seed to {fewshot_random_seed}")
+
     if seed_message:
         eval_logger.info(" | ".join(seed_message))
 
@@ -276,9 +279,6 @@ def _adjust_config(task_dict):
                         task_obj.set_config(key="num_fewshot", value=0)
                 # fewshot_random_seed set for tasks, even with a default num_fewshot (e.g. in the YAML file)
                 task_obj.set_fewshot_seed(seed=fewshot_random_seed)
-                eval_logger.info(
-                    f"Setting fewshot random generator seed to {fewshot_random_seed}"
-                )
 
                 adjusted_task_dict[task_name] = task_obj
 

lm_eval/models/vllm_vlms.py

@@ -7,9 +7,9 @@
 
 from lm_eval.api.instance import Instance
 from lm_eval.api.registry import register_model
-from lm_eval.models.utils import Collator, undistribute
+from lm_eval.models.utils import Collator, replace_placeholders, undistribute
 from lm_eval.models.vllm_causallms import VLLM
-from lm_eval.utils import simple_parse_args_string
+from lm_eval.utils import eval_logger
 
 
 try:
@@ -36,10 +36,11 @@ def __init__(
         interleave: bool = True,
         # TODO<baber>: handle max_images and limit_mm_per_prompt better
         max_images: int = 999,
-        limit_mm_per_prompt: str = "image=1",
         **kwargs,
     ):
-        kwargs["limit_mm_per_prompt"] = simple_parse_args_string(limit_mm_per_prompt)
+        if max_images != 999:
+            kwargs["limit_mm_per_prompt"] = {"image": max_images}
+            eval_logger.info(f"Setting limit_mm_per_prompt[image] to {max_images}")
         super().__init__(
             pretrained=pretrained,
             trust_remote_code=trust_remote_code,
@@ -63,6 +64,17 @@ def tok_batch_multimodal_encode(
         truncation: bool = False,
     ):
         images = [img[: self.max_images] for img in images]
+        # TODO<baber>: is the default placeholder always <image>?
+        if self.chat_applied is False:
+            strings = [
+                replace_placeholders(
+                    string,
+                    DEFAULT_IMAGE_PLACEHOLDER,
+                    DEFAULT_IMAGE_PLACEHOLDER,
+                    self.max_images,
+                )
+                for string in strings
+            ]
 
         outputs = []
         for x, i in zip(strings, images):