Fix #3931: Add unversioned flag to Package.set() and Package.set_dir()

api/python/quilt3/packages.py

@@ -34,6 +34,7 @@
     legacy_calculate_checksum,
     legacy_calculate_checksum_bytes,
     list_object_versions,
+    list_objects,
     list_url,
     put_bytes,
 )
@@ -910,11 +911,22 @@ def set_dir(self, lkey, path=None, meta=None, update_policy="incoming"):
             src_path = src.path
             if src.basename() != '':
                 src_path += '/'
-            objects, _ = list_object_versions(src.bucket, src_path)
+            try:
+                objects, _ = list_object_versions(src.bucket, src_path)
+            except botocore.exceptions.ClientError as e:
+                if e.response["Error"]["Code"] != "AccessDenied":
+                    raise
+
+                # use list_objects instead
+                print(f"list_object_versions not available; using list_objects:\n{e}")
+                objects = list_objects(src.bucket, src_path, recursive=True)
+                for obj in objects:
+                    obj["IsLatest"] = True
+
             for obj in objects:
                 if not obj['IsLatest']:
                     continue
-                # Skip S3 pseduo directory files and Keys that end in /
+                # Skip S3 pseudo-directory files and Keys that end in /
                 if obj['Key'].endswith('/'):
                     if obj['Size'] != 0:
                         warnings.warn(f'Logical keys cannot end in "/", skipping: {obj["Key"]}')

api/python/tests/integration/test_packages.py

@@ -106,6 +106,17 @@ def setup_s3_stubber_resolve_pointer_not_found(self, pkg_registry, pkg_name, *,
             }
         )
 
+    def setup_s3_stubber_cannot_list_object_versions(self, pkg_registry, pkg_name):
+        self.s3_stubber.add_client_error(
+            method="list_object_versions",
+            service_error_code="AccessDenied",
+            http_status_code=404,
+            expected_params={
+                "Bucket": pkg_registry.root.bucket,
+                "Prefix": pkg_name,
+            },
+        )
+
     def setup_s3_stubber_delete_pointer(self, pkg_registry, pkg_name, *, pointer):
         self.s3_stubber.add_response(
             method='delete_object',
@@ -631,6 +642,23 @@ def test_set_dir_wrong_update_policy(self):
             pkg.set_dir("nested", DATA_DIR, update_policy='invalid_policy')
         assert expected_err in str(e.value)
 
+    def test_set_dir_cannot_list_object_versions(self):
+        """Verify that set_dir uses list_objects if list_object_versions fails."""
+        # uses setup_s3_stubber_cannot_list_object_versions
+        registry = 's3://test-bucket'
+        pkg_registry = self.S3PackageRegistryDefault(PhysicalKey.from_url(registry))
+        pkg_name = 'Quilt/test'
+        pkg = Package()
+
+        with patch('quilt3.packages.list_object_versions', side_effect=Exception):
+            pointers = {'latest': 'latest'}
+            self.setup_s3_stubber_list_pkg_pointers(pkg_registry, pkg_name, pointers=pointers)
+            self.setup_s3_stubber_cannot_list_object_versions(pkg_registry, pkg_name)
+            pkg.set_dir(pkg_name)
+        assert pkg._meta['dir'] == pkg_name
+        assert pkg._meta['registry'] == registry
+        assert pkg._meta['physical_keys'] == ['foo', 'bar']
+
     def test_package_entry_meta(self):
         pkg = (
             Package()