diff --git a/ampm/__init__.py b/ampm/__init__.py index daab838..19b4f1d 100644 --- a/ampm/__init__.py +++ b/ampm/__init__.py @@ -1 +1 @@ -__version__ = '1.2.4' +__version__ = '1.3.0' diff --git a/ampm/cli.py b/ampm/cli.py index 33fcb9d..3a17a42 100644 --- a/ampm/cli.py +++ b/ampm/cli.py @@ -328,13 +328,23 @@ def upload( raise click.BadParameter(f'Unsupported file type: {local_path} ({local_path.stat().st_type})') else: if compressed: - raise click.BadParameter('NOT IMPLEMENTED: Compressed artifacts with remote paths (add `--uncompressed`)') + if remote_path.endswith('.tar.gz'): + artifact_type = 'tar.gz' + name = name or remote_path.strip('/').split('/')[-1][:-len('.tar.gz')] + elif remote_path.endswith('.gz'): + artifact_type = 'gz' + name = name or remote_path.strip('/').split('/')[-1][:-len('.gz')] + else: + raise click.BadParameter(f'Remote artifact is not compressed using a known compression method ' + f'(.tar.gz or .gz): {remote_path}\n' + f'Try adding `--uncompressed` to create an uncompressed artifact.') + else: + artifact_type = 'file' + name = name or remote_path.strip('/').split('/')[-1] - name = name or remote_path.strip('/').split('/')[-1] try: - artifact_type = 'file' artifact_hash = remote_repo.hash_remote_file(remote_path, progress_bar=True) - except IsADirectoryError: + except OSError: artifact_type = 'dir' artifact_hash = None diff --git a/tests/test_cli.py b/tests/test_cli.py index 0caf4a8..dee8040 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ import json import gzip import re +import tarfile import time import pytest import ampm.cli @@ -12,7 +13,7 @@ @pytest.fixture() def upload(nfs_repo_uri): def _upload( - local_path: str, + local_path: Optional[str], artifact_type: str, compressed: bool, attributes=None, @@ -22,7 +23,9 @@ def _upload( attributes = {} runner = CliRunner(mix_stderr=False, env={'AMPM_SERVER': nfs_repo_uri}) - args = ['upload', local_path, '--type', artifact_type, '--compressed' if compressed else '--uncompressed'] + args = ['upload', '--type', artifact_type, '--compressed' if compressed else '--uncompressed'] + if local_path: + args += [local_path] if remote_path: args += ['--remote-path', remote_path] for k, v in attributes.items(): @@ -169,6 +172,90 @@ def test_upload_dir_location(nfs_repo_path, nfs_mount_path, clean_repos, upload, == b"boo\n", "File nested has wrong contents" +@pytest.mark.parametrize('is_compressed', ['compressed', 'uncompressed']) +def test_upload_external_single_file(nfs_repo_path, nfs_mount_path: Path, clean_repos, upload, is_compressed): + _ = clean_repos + remote_path = nfs_mount_path / 'custom_dir' / ('foobar.txt' + ('.gz' if is_compressed == 'compressed' else '')) + + data = b'hello\n' + if is_compressed == 'compressed': + file_contents = gzip.compress(data) + else: + file_contents = data + + remote_path.parent.mkdir(parents=True, exist_ok=True) + remote_path.write_bytes(file_contents) + + artifact_hash = upload( + local_path=None, + artifact_type='foo', + remote_path=str(remote_path), + compressed=is_compressed == 'compressed' + ) + assert (nfs_repo_path / 'metadata' / 'foo' / f'{artifact_hash}.toml').is_file(), "Metadata file wasn't created" + assert remote_path.read_bytes() == file_contents, "Data file was modified/deleted!!!" + + +def test_upload_external_dir_ok(nfs_repo_path, nfs_mount_path: Path, clean_repos, upload): + _ = clean_repos + remote_path = nfs_mount_path / 'custom_dir' / 'foo_dir' + + data = b'hello\n' + remote_path.mkdir(parents=True, exist_ok=True) + (remote_path / 'a.txt').write_bytes(data) + + artifact_hash = upload( + local_path=None, + artifact_type='foo', + remote_path=str(remote_path), + compressed=False + ) + assert (nfs_repo_path / 'metadata' / 'foo' / f'{artifact_hash}.toml').is_file(), "Metadata file wasn't created" + assert remote_path.is_dir(), "Data dir was modified/deleted!!!" + + +def test_upload_external_dir_err(nfs_repo_path, nfs_mount_path: Path, clean_repos, upload): + _ = clean_repos + remote_path = nfs_mount_path / 'custom_dir' / 'foo_dir' + + data = b'hello\n' + remote_path.mkdir(parents=True, exist_ok=True) + (remote_path / 'a.txt').write_bytes(data) + + with pytest.raises(AssertionError): + _artifact_hash = upload( + local_path=None, + artifact_type='foo', + remote_path=str(remote_path), + compressed=True + ) + assert not (nfs_repo_path / 'metadata' / 'foo').is_dir(), "Metadata file was created" + + +def test_upload_external_archive(nfs_repo_path, nfs_mount_path: Path, clean_repos, upload): + _ = clean_repos + tmp_remote_path = nfs_mount_path / 'custom_dir' / 'a.txt' + remote_path = nfs_mount_path / 'custom_dir' / 'foo_dir.tar.gz' + + data = b'hello\n' + tmp_remote_path.parent.mkdir(parents=True, exist_ok=True) + tmp_remote_path.write_bytes(data) + + with tarfile.open(str(remote_path), 'w:gz') as tar: + tar.add(str(tmp_remote_path), arcname='a.txt') + + file_contents = remote_path.read_bytes() + + artifact_hash = upload( + local_path=None, + artifact_type='foo', + remote_path=str(remote_path), + compressed=True + ) + assert (nfs_repo_path / 'metadata' / 'foo' / f'{artifact_hash}.toml').is_file(), "Metadata file wasn't created" + assert remote_path.read_bytes() == file_contents, "Data archive was modified/deleted!!!" + + @pytest.mark.parametrize('is_compressed', ['compressed', 'uncompressed']) def test_download_single_file(clean_repos, upload, download, is_compressed): _ = clean_repos @@ -208,6 +295,70 @@ def test_download_dir(clean_repos, upload, download, is_compressed): assert (artifact_path / 'nested' / 'boo.txt').read_bytes() == b"boo\n", "File nested has wrong contents" +@pytest.mark.parametrize('is_compressed', ['compressed', 'uncompressed']) +def test_download_external_single_file(nfs_mount_path: Path, clean_repos, upload, download, is_compressed): + _ = clean_repos + remote_path = nfs_mount_path / 'custom_dir' / ('foobar.txt' + ('.gz' if is_compressed == 'compressed' else '')) + + data = b'hello\n' + if is_compressed == 'compressed': + file_contents = gzip.compress(data) + else: + file_contents = data + + remote_path.parent.mkdir(parents=True, exist_ok=True) + remote_path.write_bytes(file_contents) + + artifact_hash = upload( + local_path=None, + artifact_type='foo', + remote_path=str(remote_path), + compressed=is_compressed == 'compressed' + ) + artifact_path = download(f'foo:{artifact_hash}', {}) + assert artifact_path.read_bytes() == data, "Downloaded file has wrong contents" + + +def test_download_external_dir_ok(nfs_mount_path: Path, clean_repos, upload, download): + _ = clean_repos + remote_path = nfs_mount_path / 'custom_dir' / 'foo_dir' + + file_contents = b'hello\n' + remote_path.mkdir(parents=True, exist_ok=True) + (remote_path / 'a.txt').write_bytes(file_contents) + + artifact_hash = upload( + local_path=None, + artifact_type='foo', + remote_path=str(remote_path), + compressed=False + ) + artifact_path = download(f'foo:{artifact_hash}', {}) + assert (artifact_path / 'a.txt').read_bytes() == file_contents, "Downloaded file has wrong contents" + + +def test_download_external_archive(nfs_mount_path: Path, clean_repos, upload, download): + _ = clean_repos + tmp_remote_path = nfs_mount_path / 'custom_dir' / 'a.txt' + remote_path = nfs_mount_path / 'custom_dir' / 'foo_dir.tar.gz' + + data = b'hello\n' + tmp_remote_path.parent.mkdir(parents=True, exist_ok=True) + tmp_remote_path.write_bytes(data) + + with tarfile.open(str(remote_path), 'w:gz') as tar: + tar.add(str(tmp_remote_path), arcname='a.txt') + + artifact_hash = upload( + local_path=None, + artifact_type='foo', + remote_path=str(remote_path), + compressed=True + ) + artifact_path = download(f'foo:{artifact_hash}', {}) + assert (artifact_path / 'a.txt').read_bytes() == data, "Downloaded file has wrong contents" + + @pytest.mark.parametrize('filter_type', ['num', 'date', 'semver']) def test_attr_filters(clean_repos, upload, list_, filter_type): _ = clean_repos