Add reference parquet files for pyarrow>=1 (#416)

JDASoftwareGroup · Feb 17, 2021 · 6514c1f · 6514c1f
1 parent 6dc085a
commit 6514c1f
Show file tree

Hide file tree

Showing 6 changed files with 27 additions and 3 deletions.
diff --git a/reference-data/arrow-compat/0.17.1.parquet b/reference-data/arrow-compat/0.17.1.parquet
diff --git a/reference-data/arrow-compat/1.0.1.parquet b/reference-data/arrow-compat/1.0.1.parquet
diff --git a/reference-data/arrow-compat/2.0.0.parquet b/reference-data/arrow-compat/2.0.0.parquet
diff --git a/reference-data/arrow-compat/3.0.0.parquet b/reference-data/arrow-compat/3.0.0.parquet
diff --git a/reference-data/arrow-compat/batch_generate_references.sh b/reference-data/arrow-compat/batch_generate_references.sh
@@ -2,10 +2,10 @@
 
 # Note: this assumes you have kartothek installed in your current environment and you are using conda
 
-PYARROW_VERSIONS="0.14.1 0.15.0 0.16.0 1.0.0"
+PYARROW_VERSIONS="0.14.1 0.15.0 0.16.0 0.17.1 1.0.1 2.0.0 3.0.0"
 
 for pyarrow_version in $PYARROW_VERSIONS; do
-    echo $pyarrow_version 
+    echo $pyarrow_version
     conda install -y pyarrow==$pyarrow_version
     ./generate_reference.py || (echo "Failed for version $pyarrow_version"; exit 1)
 done
diff --git a/tests/serialization/test_arrow_compat.py b/tests/serialization/test_arrow_compat.py
@@ -11,8 +11,21 @@
 from kartothek.core.testing import get_dataframe_alltypes
 from kartothek.serialization import ParquetSerializer
 
+KNOWN_ARROW_VERSIONS = [
+    "0.12.1",
+    "0.13.0",
+    "0.14.1",
+    "0.15.0",
+    "0.16.0",
+    "0.17.1",
+    "1.0.0",
+    "1.0.1",
+    "2.0.0",
+    "3.0.0",
+]
 
-@pytest.fixture(params=["0.12.1", "0.13.0", "0.14.1", "0.15.0", "0.16.0", "1.0.0"])
+
+@pytest.fixture(params=KNOWN_ARROW_VERSIONS)
 def arrow_version(request):
     yield request.param
 
@@ -28,6 +41,17 @@ def reference_store():
     return get_store_from_url("hfs://{}".format(path))
 
 
+def test_current_arrow_version_tested():
+    """Ensure that we do not forget to generate the reference file"""
+    import pyarrow as pa
+    from packaging.version import parse
+
+    version = parse(pa.__version__)
+    is_stable = not version.is_devrelease and not version.is_prerelease
+    if is_stable:
+        assert pa.__version__ in KNOWN_ARROW_VERSIONS
+
+
 def test_arrow_compat(arrow_version, reference_store, mocker):
     """
     Test if reading/writing across the supported arrow versions is actually