Fix loading without limit (#1594)

Signed-off-by: elronbandel <[email protected]>
IBM · Feb 10, 2025 · 7d3d517 · 7d3d517
1 parent 82a440f
commit 7d3d517
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 6 deletions.
diff --git a/src/unitxt/loaders.py b/src/unitxt/loaders.py
@@ -355,10 +355,13 @@ def split_generator(self, split: str) -> Generator:
             dataset = self.filter_load(dataset)
 
         limit = self.get_limit()
-        for i, instance in enumerate(dataset):
-            yield instance
-            if i + 1 >= limit:
-                break
+        if limit is None:
+            yield from dataset
+        else:
+            for i, instance in enumerate(dataset):
+                yield instance
+                if i + 1 >= limit:
+                    break
 
 
 class LoadCSV(LazyLoader):

diff --git a/utils/.secrets.baseline b/utils/.secrets.baseline
@@ -151,7 +151,7 @@
         "filename": "src/unitxt/loaders.py",
         "hashed_secret": "840268f77a57d5553add023cfa8a4d1535f49742",
         "is_verified": false,
-        "line_number": 589,
+        "line_number": 592,
         "is_secret": false
       }
     ],
@@ -184,5 +184,5 @@
       }
     ]
   },
-  "generated_at": "2025-02-10T13:25:07Z"
+  "generated_at": "2025-02-10T14:10:51Z"
 }