split whole repo path into mount path and actual repo path, make `--r…

…emote-path` be an absolute path

.github/workflows/CI.yml

@@ -6,23 +6,23 @@ on:
   pull_request:
 
 jobs:
-  build-linux:
-    name: Build linux binaries
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Build
-        run: ./build.sh
-      - name: Upload results
-        uses: actions/upload-artifact@v3
-        with:
-          name: ampm-linux-binaries
-          if-no-files-found: error
-          path: |
-            dist/ampm.tar.gz
-            dist/get_ampm.sh
+#  build-linux:
+#    name: Build linux binaries
+#    runs-on: ubuntu-latest
+#    timeout-minutes: 10
+#    steps:
+#      - name: Checkout code
+#        uses: actions/checkout@v2
+#      - name: Build
+#        run: ./build.sh
+#      - name: Upload results
+#        uses: actions/upload-artifact@v3
+#        with:
+#          name: ampm-linux-binaries
+#          if-no-files-found: error
+#          path: |
+#            dist/ampm.tar.gz
+#            dist/get_ampm.sh
   test-linux:
     name: Test linux binaries
     runs-on: ubuntu-latest

ampm/__init__.py

@@ -1 +1 @@
-__version__ = '1.2.0'
+__version__ = '1.2.2'

ampm/get_ampm.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-REMOTE_REPO="nfs://127.0.0.1/mnt/myshareddir"
+REMOTE_REPO="nfs://127.0.0.1/mnt/myshareddir#ampm_repo"
 
 if [ "$(id -u)" -ne 0 ]; then
     echo 'This script must be run as root'

ampm/repo/nfs.py

@@ -333,15 +333,16 @@ def chunked():
 
 
 class NfsRepo(ArtifactRepo):
-    def __init__(self, host: str, remote_path: str):
+    def __init__(self, host: str, mount_path: str, repo_path: str):
         self.host = host
-        self.remote_path = remote_path
+        self.mount_path = mount_path
+        self.repo_path = repo_path
         self.connection: Optional[NfsConnection] = None
 
     @contextlib.contextmanager
     def _connected(self) -> ContextManager["NfsConnection"]:
         if self.connection is None:
-            with NfsConnection.connect(self.host, self.remote_path) as nfs:
+            with NfsConnection.connect(self.host, self.mount_path) as nfs:
                 self.connection = nfs
                 try:
                     yield nfs
@@ -352,11 +353,12 @@ def _connected(self) -> ContextManager["NfsConnection"]:
 
     @staticmethod
     def from_uri_part(uri_part: str) -> "NfsRepo":
-        host, remote_path = uri_part.split("/", 1)
-        return NfsRepo(host, '/' + remote_path)
+        uri_part, repo_path = uri_part.split("#", 1)
+        host, mount_path = uri_part.split("/", 1)
+        return NfsRepo(host, '/' + mount_path.strip('/'), repo_path.strip('/'))
 
     def into_uri(self) -> str:
-        return f"nfs://{self.host}/{self.remote_path.lstrip('/')}"
+        return f"nfs://{self.host}/{self.mount_path.lstrip('/')}#{self.repo_path}"
 
     def upload(self, metadata: ArtifactMetadata, local_path: Optional[Path]):
         assert metadata.path_type in ARTIFACT_TYPES, f'Invalid artifact path type: {metadata.path_type}'
@@ -489,22 +491,21 @@ def hash_remote_file(self, remote_path: str, progress_bar=False) -> str:
                 hasher.update(chunk)
             return hasher.hexdigest()
 
-    @staticmethod
-    def metadata_path_of(artifact_type: str, artifact_hash: str, suffix='.toml') -> str:
-        return f'metadata/{artifact_type}/{artifact_hash or ""}{suffix}'
+    def metadata_path_of(self, artifact_type: str, artifact_hash: str, suffix='.toml') -> str:
+        return f'{self.repo_path}/metadata/{artifact_type}/{artifact_hash or ""}{suffix}'
 
-    @staticmethod
-    def artifact_base_path_of(metadata: ArtifactMetadata, suffix='') -> str:
+    def artifact_base_path_of(self, metadata: ArtifactMetadata, suffix='') -> str:
         if metadata.path_location:
-            return metadata.path_location
+            assert metadata.path_location.startswith(self.mount_path)
+            return metadata.path_location[len(self.mount_path):].lstrip('/')
         else:
-            return f'artifacts/{metadata.type.lower()}/{metadata.hash.lower()}{suffix}'
+            return f'{self.repo_path}/artifacts/{metadata.type.lower()}/{metadata.hash.lower()}{suffix}'
 
-    @staticmethod
-    def artifact_path_of(metadata: ArtifactMetadata, suffix='') -> str:
+    def artifact_path_of(self, metadata: ArtifactMetadata, suffix='') -> str:
         if metadata.path_location:
-            return metadata.path_location
+            assert metadata.path_location.startswith(self.mount_path)
+            return metadata.path_location[len(self.mount_path):].lstrip('/')
         else:
-            return f'artifacts/{metadata.type.lower()}/{metadata.hash.lower()}{suffix}/' \
+            return f'{self.repo_path}/artifacts/{metadata.type.lower()}/{metadata.hash.lower()}{suffix}/' \
                    f'{metadata.name}{metadata.path_suffix}'
 

tests/conftest.py

@@ -11,7 +11,12 @@
 
 @pytest.fixture()
 def nfs_repo_path(nfs_repo) -> Path:
-    return Path(nfs_repo.remote_path)
+    return Path(nfs_repo.mount_path) / nfs_repo.repo_path
+
+
+@pytest.fixture()
+def nfs_mount_path(nfs_repo) -> Path:
+    return Path(nfs_repo.mount_path)
 
 
 @pytest.fixture()
@@ -60,5 +65,5 @@ def nfs_repo(nfs_server) -> NfsRepo:
     random_id = randbytes(8).hex()
     nfs_root = nfs_server["root"] / random_id
     nfs_root.mkdir()
-    return NfsRepo.from_uri_part(f'{nfs_server["host"]}{nfs_root}')
+    return NfsRepo.from_uri_part(f'{nfs_server["host"]}{nfs_root}#repo')
 

tests/test_cli.py

@@ -93,26 +93,28 @@ def test_upload_single_file(nfs_repo_path, clean_repos, upload, is_compressed):
 
 
 @pytest.mark.parametrize('is_compressed', ['compressed', 'uncompressed'])
-def test_upload_single_file_location(nfs_repo_path, clean_repos, upload, is_compressed):
+def test_upload_single_file_location(nfs_repo_path, nfs_mount_path, clean_repos, upload, is_compressed):
     _ = clean_repos
     artifact_hash = upload(
         'tests/dummy_data/foobar.txt',
         artifact_type='foo',
-        remote_path='/custom_dir/foobar.txt' + ('.gz' if is_compressed == 'compressed' else ''),
+        remote_path=str(
+            nfs_mount_path / 'custom_dir' / ('foobar.txt' + ('.gz' if is_compressed == 'compressed' else ''))
+        ),
         compressed=is_compressed == 'compressed'
     )
     assert (nfs_repo_path / 'metadata' / 'foo' / f'{artifact_hash}.toml').is_file(), "Metadata file wasn't created"
     if is_compressed == 'compressed':
-        assert (nfs_repo_path / 'custom_dir' / 'foobar.txt.gz').is_file(), \
+        assert (nfs_mount_path / 'custom_dir' / 'foobar.txt.gz').is_file(), \
             "Data file wasn't created"
         decompressed = gzip.decompress(
-            (nfs_repo_path / 'custom_dir' / 'foobar.txt.gz').read_bytes()
+            (nfs_mount_path / 'custom_dir' / 'foobar.txt.gz').read_bytes()
         )
         assert decompressed == b'foo bar\n', "Data file has wrong contents"
     else:
-        assert (nfs_repo_path / 'custom_dir' / 'foobar.txt').is_file(), \
+        assert (nfs_mount_path / 'custom_dir' / 'foobar.txt').is_file(), \
             "Data file wasn't created"
-        assert (nfs_repo_path / 'custom_dir' / 'foobar.txt').read_bytes() == b'foo bar\n', \
+        assert (nfs_mount_path / 'custom_dir' / 'foobar.txt').read_bytes() == b'foo bar\n', \
             "Data file has wrong contents"
 
 
@@ -141,27 +143,29 @@ def test_upload_dir(nfs_repo_path, clean_repos, upload, is_compressed):
 
 
 @pytest.mark.parametrize('is_compressed', ['compressed', 'uncompressed'])
-def test_upload_dir_location(nfs_repo_path, clean_repos, upload, is_compressed):
+def test_upload_dir_location(nfs_repo_path, nfs_mount_path, clean_repos, upload, is_compressed):
     _ = clean_repos
     artifact_hash = upload(
         'tests/dummy_data/foo_dir',
         artifact_type='foo',
-        remote_path='/custom_dir/foo_dir' + ('.tar.gz' if is_compressed == 'compressed' else ''),
+        remote_path=str(
+            nfs_mount_path / 'custom_dir' / ('foo_dir' + ('.tar.gz' if is_compressed == 'compressed' else ''))
+        ),
         compressed=is_compressed == 'compressed'
     )
     assert (nfs_repo_path / 'metadata' / 'foo' / f'{artifact_hash}.toml').is_file(), \
         "Metadata file wasn't created"
     if is_compressed == 'compressed':
-        assert (nfs_repo_path / 'custom_dir' / 'foo_dir.tar.gz').is_file(), \
+        assert (nfs_mount_path / 'custom_dir' / 'foo_dir.tar.gz').is_file(), \
             "Archive wasn't created"
     else:
-        assert (nfs_repo_path / 'custom_dir' / 'foo_dir').is_dir(), \
+        assert (nfs_mount_path / 'custom_dir' / 'foo_dir').is_dir(), \
             "Dir wasn't created"
-        assert (nfs_repo_path / 'custom_dir' / 'foo_dir' / 'hello.txt').is_file(), \
+        assert (nfs_mount_path / 'custom_dir' / 'foo_dir' / 'hello.txt').is_file(), \
             "File inside wasn't created"
-        assert (nfs_repo_path / 'custom_dir' / 'foo_dir' / 'nested' / 'boo.txt').is_file(), \
+        assert (nfs_mount_path / 'custom_dir' / 'foo_dir' / 'nested' / 'boo.txt').is_file(), \
             "File nested inside wasn't created"
-        assert (nfs_repo_path / 'custom_dir' / 'foo_dir' / 'nested' / 'boo.txt').read_bytes() \
+        assert (nfs_mount_path / 'custom_dir' / 'foo_dir' / 'nested' / 'boo.txt').read_bytes() \
                == b"boo\n", "File nested has wrong contents"
 
 
@@ -178,12 +182,12 @@ def test_download_single_file(clean_repos, upload, download, is_compressed):
 
 
 @pytest.mark.parametrize('is_compressed', ['compressed', 'uncompressed'])
-def test_download_single_file_location(clean_repos, upload, download, is_compressed):
+def test_download_single_file_location(nfs_mount_path, clean_repos, upload, download, is_compressed):
     _ = clean_repos
     artifact_hash = upload(
         'tests/dummy_data/foobar.txt',
         artifact_type='foo',
-        remote_path='/custom_dir/foobar.txt',
+        remote_path=str(nfs_mount_path / 'custom_dir' / 'foobar.txt'),
         compressed=is_compressed == 'compressed'
     )
     artifact_path = download(f'foo:{artifact_hash}', {})

tests/test_nfs.py

@@ -5,10 +5,10 @@
 from ampm.utils import randbytes
 
 
-def test_operations(clean_repos, nfs_repo: NfsRepo, nfs_repo_path: Path):
+def test_operations(clean_repos, nfs_repo: NfsRepo, nfs_mount_path: Path):
     _ = clean_repos
 
-    with NfsConnection.connect(nfs_repo.host, nfs_repo.remote_path) as nfs:
+    with NfsConnection.connect(nfs_repo.host, nfs_repo.mount_path) as nfs:
         ident = f'ampm_tests_{randbytes(8).hex()}'
         local_path = Path(f'/tmp/{ident}')
         remote_path = Path(f'nfs_tests')
@@ -17,8 +17,8 @@ def test_operations(clean_repos, nfs_repo: NfsRepo, nfs_repo_path: Path):
             local_path.mkdir(parents=True, exist_ok=True)
             (local_path / 'foo.txt').write_text('foo bar')
             nfs.upload(local_path / 'foo.txt', str(remote_path / 'foo.txt'))
-            assert (nfs_repo_path / remote_path / 'foo.txt').is_file(), 'Uploaded file missing'
-            assert (nfs_repo_path / remote_path / 'foo.txt').read_text() == 'foo bar', 'Uploaded file content mismatch'
+            assert (nfs_mount_path / remote_path / 'foo.txt').is_file(), 'Uploaded file missing'
+            assert (nfs_mount_path / remote_path / 'foo.txt').read_text() == 'foo bar', 'Uploaded file content mismatch'
 
             # List
             assert sorted(list(nfs.list_dir(str(remote_path)))) == [b'.', b'..', b'foo.txt'], \
@@ -35,12 +35,12 @@ def test_operations(clean_repos, nfs_repo: NfsRepo, nfs_repo_path: Path):
                 'List dir mismatch after rename'
 
             # Symlink
-            nfs.symlink(str(nfs_repo_path / remote_path / 'foo2.txt'), str(remote_path / 'foo3.txt'))
+            nfs.symlink(str(nfs_mount_path / remote_path / 'foo2.txt'), str(remote_path / 'foo3.txt'))
             assert sorted(list(nfs.list_dir(str(remote_path)))) == [b'.', b'..', b'foo2.txt', b'foo3.txt'], \
                 'List dir mismatch after symlink'
 
             # Readlink
             assert nfs.readlink(str(remote_path / 'foo3.txt')) \
-                   == str(nfs_repo_path / remote_path / 'foo2.txt').encode(), 'Readlink mismatch'
+                   == str(nfs_mount_path / remote_path / 'foo2.txt').encode(), 'Readlink mismatch'
         finally:
             shutil.rmtree(local_path, ignore_errors=True)