adjust to new readers for csv

Signed-off-by: dafnapension <[email protected]>
IBM · Jan 26, 2025 · e00617f · e00617f
1 parent ef29521
commit e00617f
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 4 deletions.
diff --git a/src/unitxt/loaders.py b/src/unitxt/loaders.py
@@ -447,9 +447,13 @@ def load_iterables(self):
         return iterables
 
     def split_generator(self, split: str) -> Generator:
-        dataset = pd.read_csv(
-            self.files[split], nrows=self.get_limit(), sep=self.sep
-        ).to_dict("records")
+        dataset = self.__class__._loader_cache.get(str(self) + "_" + split, None)
+        if dataset is None:
+            reader = self.get_reader()
+            dataset = reader(self.files[split], **self.get_args()).to_dict("records")
+            self.__class__._loader_cache.max_size = settings.loader_cache_size
+            self.__class__._loader_cache[str(self) + "_" + split] = dataset
+
         yield from dataset
 
 

diff --git a/utils/.secrets.baseline b/utils/.secrets.baseline
@@ -151,7 +151,7 @@
         "filename": "src/unitxt/loaders.py",
         "hashed_secret": "840268f77a57d5553add023cfa8a4d1535f49742",
         "is_verified": false,
-        "line_number": 595,
+        "line_number": 599,
         "is_secret": false
       }
     ],