Merge pull request #753 from roflcoopter/feature/tier_handler_instant…

…_delete

Delete files instead of moving them

docker/Dockerfile

@@ -9,11 +9,17 @@ FROM roflcoopter/${ARCH}-ffmpeg:${FFMPEG_VERSION} as ffmpeg
 FROM roflcoopter/${ARCH}-wheels:${WHEELS_VERSION} as wheels
 
 # Build GPAC
-FROM ubuntu:${UBUNTU_VERSION} AS gpac
+FROM roflcoopter/${ARCH}-base:${BASE_VERSION} as gpac
 
 ENV \
   DEBIAN_FRONTEND=noninteractive
 
+RUN \
+  if [ "$ARCH" = "armhf" ] || \
+  [ "$ARCH" = "rpi3" ] || \
+  [ "$ARCH" = "aarch64" ] || \
+  [ "$ARCH" = "jetson-nano" ]; then echo "Crossbuilding!" && cross-build-start; fi
+
 RUN \
   apt-get update && apt-get install -y --no-install-recommends \
   build-essential \

rootfs/etc/cont-init.d/40-set-env-vars

@@ -49,7 +49,7 @@ export HOME=/home/abc
 printf "/home/abc" > /var/run/environment/HOME
 
 # Find latest version of postgresql
-export PG_VERSION=$(pg_config --version | awk '{print $2}' | awk -F'.' '{print $1}')
+export PG_VERSION=$(psql --version | awk '{print $3}' | awk -F'.' '{print $1}')
 export PG_BIN="/usr/lib/postgresql/$PG_VERSION/bin"
 printf "$PG_VERSION" > /var/run/environment/PG_VERSION
 printf "$PG_BIN" > /var/run/environment/PG_BIN

tests/components/storage/test_tier_handler.py

@@ -6,9 +6,20 @@
 import pytest
 from sqlalchemy import select
 
-from viseron.components.storage.const import CONFIG_RECORDER
+from viseron import Viseron
+from viseron.components.storage import Storage
+from viseron.components.storage.const import (
+    COMPONENT as STORAGE_COMPONENT,
+    CONFIG_RECORDER,
+)
 from viseron.components.storage.models import Recordings
-from viseron.components.storage.tier_handler import SegmentsTierHandler, handle_file
+from viseron.components.storage.tier_handler import (
+    RecordingsTierHandler,
+    SegmentsTierHandler,
+    ThumbnailTierHandler,
+    find_next_tier_segments,
+    handle_file,
+)
 from viseron.domains.camera.const import CONFIG_LOOKBACK
 
 from tests.common import BaseTestWithRecordings
@@ -54,15 +65,24 @@ def test_handle_file_move(
 
 
 @dataclass
-class MockQueryResult:
+class MockRecordingsQueryResult:
     """Mock query result."""
 
-    recording_id: int
+    recording_id: int | None
     file_id: int
     path: str
     tier_path: str
 
 
+@dataclass
+class MockFilesQueryResult:
+    """Mock query result."""
+
+    id: int
+    path: str
+    tier_path: str
+
+
 def _get_tier_config(events: bool, continuous: bool):
     """Get tier config for test."""
     max_age_events = None
@@ -139,12 +159,12 @@ def test__check_tier(
             "viseron.components.storage.tier_handler.handle_file"
         ):
             mock_get_recordings_to_move.return_value = [
-                MockQueryResult(1, 1, "/tmp/test1.mp4", "/tmp/"),
-                MockQueryResult(1, 2, "/tmp/test2.mp4", "/tmp/"),
+                MockRecordingsQueryResult(1, 1, "/tmp/test1.mp4", "/tmp/"),
+                MockRecordingsQueryResult(1, 2, "/tmp/test2.mp4", "/tmp/"),
             ]
             mock_get_files_to_move.return_value = [
-                MockQueryResult(1, 1, "/tmp/test1.mp4", "/tmp/"),
-                MockQueryResult(1, 2, "/tmp/test2.mp4", "/tmp/"),
+                MockFilesQueryResult(1, "/tmp/test1.mp4", "/tmp/"),
+                MockFilesQueryResult(2, "/tmp/test2.mp4", "/tmp/"),
             ]
             tier_handler._check_tier(  # pylint: disable=protected-access
                 self._get_db_session
@@ -167,3 +187,203 @@ def test__check_tier(
             recordings = session.execute(stmt).scalars().fetchall()
             assert len(recordings) == recordings_amount
             assert recordings[0].id == first_recording_id
+
+    @pytest.mark.parametrize(
+        "tiers_config, recording_id, force_delete, next_tier_index, "
+        "move_thumbnail_called, move_event_clip_called",
+        [
+            (  # Test that check_tier deletes the file if next tier is None
+                [_get_tier_config(events=True, continuous=True)],
+                1,
+                True,
+                None,
+                True,
+                True,
+            ),
+            # Test that check_tier deletes the file if its not part of a recording and
+            # next tier does not store continuous
+            (
+                [
+                    _get_tier_config(events=True, continuous=True),
+                    _get_tier_config(events=True, continuous=False),
+                ],
+                None,
+                True,
+                None,
+                False,
+                False,
+            ),
+            # Test that check_tier moves the file if its part of a recording and
+            # the next tier stores events
+            (
+                [
+                    _get_tier_config(events=True, continuous=True),
+                    _get_tier_config(events=True, continuous=False),
+                ],
+                1,
+                False,
+                1,
+                True,
+                True,
+            ),
+            # Test that check_tier moves the file to the correct tier when the next tier
+            # does not store events but the next next tier does
+            (
+                [
+                    _get_tier_config(events=True, continuous=True),
+                    _get_tier_config(events=False, continuous=False),
+                    _get_tier_config(events=True, continuous=False),
+                    _get_tier_config(events=False, continuous=True),
+                ],
+                1,
+                False,
+                2,
+                True,
+                True,
+            ),
+        ],
+    )
+    def test__check_tier_next_tier4(
+        self,
+        vis: Viseron,
+        tiers_config,
+        recording_id: int,
+        force_delete: bool,
+        next_tier_index: int | None,
+        move_thumbnail_called: bool,
+        move_event_clip_called: bool,
+    ):
+        """Test that check_tier finds the correct tier."""
+        mock_camera = Mock()
+        mock_camera.identifier = "test"
+        mock_camera.config = {CONFIG_RECORDER: {CONFIG_LOOKBACK: 5}}
+
+        tier_handlers = []
+        for i, tier_config in enumerate(tiers_config):
+            tier_handler = SegmentsTierHandler(
+                vis,
+                mock_camera,
+                i,
+                "recorder",
+                "segments",
+                tier_config,
+                None,
+            )
+            tier_handlers.append(tier_handler)
+        recordings_tier_handler = MagicMock(spec=RecordingsTierHandler)
+        thumbnail_tier_handler = MagicMock(spec=ThumbnailTierHandler)
+        vis.data[STORAGE_COMPONENT].camera_tier_handlers = {
+            "test": {
+                "recorder": [
+                    {
+                        "segments": tier_handler,
+                        "thumbnails": thumbnail_tier_handler,
+                        "recordings": recordings_tier_handler,
+                    }
+                    for tier_handler in tier_handlers
+                ]
+            }
+        }
+
+        with patch(
+            "viseron.components.storage.tier_handler.get_recordings_to_move"
+        ) as mock_get_recordings_to_move, patch(
+            "viseron.components.storage.tier_handler.get_files_to_move"
+        ) as mock_get_files_to_move, patch(
+            "viseron.components.storage.tier_handler.handle_file"
+        ) as mock_handle_file:
+            mock_get_recordings_to_move.return_value = [
+                MockRecordingsQueryResult(recording_id, 1, "/tmp/test1.mp4", "/tmp/"),
+            ]
+            mock_get_files_to_move.return_value = [
+                MockFilesQueryResult(1, "/tmp/test1.mp4", "/tmp/"),
+            ]
+            tier_handlers[0]._check_tier(  # pylint: disable=protected-access
+                self._get_db_session
+            )
+            mock_handle_file.assert_called_once_with(
+                self._get_db_session,
+                tier_handlers[0]._storage,  # pylint: disable=protected-access
+                tier_handlers[0]._camera.identifier,  # pylint: disable=protected-access
+                tier_handlers[0].tier,
+                tier_handlers[next_tier_index].tier if next_tier_index else None,
+                "/tmp/test1.mp4",
+                "/tmp/",
+                tier_handlers[0]._logger,  # pylint: disable=protected-access
+                force_delete=force_delete,
+            )
+            if move_thumbnail_called:
+                thumbnail_tier_handler.move_thumbnail.assert_called_once_with(
+                    1, tier_handlers[next_tier_index].tier if next_tier_index else None
+                )
+            if move_event_clip_called:
+                recordings_tier_handler.move_event_clip.assert_called_once_with(
+                    1, tier_handlers[next_tier_index].tier if next_tier_index else None
+                )
+
+
+def test_find_next_tier_segments(vis: Viseron):
+    """Test find_next_tier_segments."""
+    mock_storage = Mock(spec=Storage)
+    mock_camera = Mock()
+    mock_camera.identifier = "test_camera"
+    mock_camera.config = {CONFIG_RECORDER: {CONFIG_LOOKBACK: 5}}
+
+    tier_handler_0 = SegmentsTierHandler(
+        vis,
+        mock_camera,
+        0,
+        "recorder",
+        "segments",
+        _get_tier_config(events=True, continuous=True),
+        None,
+    )
+    tier_handler_1 = SegmentsTierHandler(
+        vis,
+        mock_camera,
+        1,
+        "recorder",
+        "segments",
+        _get_tier_config(events=False, continuous=False),
+        None,
+    )
+    tier_handler_2 = SegmentsTierHandler(
+        vis,
+        mock_camera,
+        2,
+        "recorder",
+        "segments",
+        _get_tier_config(events=True, continuous=False),
+        None,
+    )
+
+    tier_handler_3 = SegmentsTierHandler(
+        vis,
+        mock_camera,
+        3,
+        "recorder",
+        "segments",
+        _get_tier_config(events=False, continuous=True),
+        None,
+    )
+
+    mock_camera.identifier = "test_camera"
+    mock_storage.camera_tier_handlers = {
+        "test_camera": {
+            "recorder": [
+                {"segments": tier_handler_0},
+                {"segments": tier_handler_1},
+                {"segments": tier_handler_2},
+                {"segments": tier_handler_3},
+            ]
+        }
+    }
+
+    result = find_next_tier_segments(mock_storage, 0, mock_camera, "events")
+    assert result == tier_handler_2
+
+    result = find_next_tier_segments(mock_storage, 0, mock_camera, "continuous")
+    assert result == tier_handler_3
+
+    result = find_next_tier_segments(mock_storage, 2, mock_camera, "events")
+    assert result is None

tests/components/webserver/common.py

@@ -81,8 +81,8 @@ def setUp(self) -> None:
 
     def tearDown(self) -> None:
         """Tear down the test."""
+        super().tearDown()
         self.vis.shutdown()
-        return super().tearDown()
 
     def get_app(self):
         """Get the application.

viseron/__init__.py

@@ -536,7 +536,6 @@ def join(
         | multiprocessing.process.BaseProcess,
     ) -> None:
         thread_or_process.join(timeout=10)
-        time.sleep(0.5)  # Wait for process to exit properly
         if thread_or_process.is_alive():
             LOGGER.error(f"{thread_or_process.name} did not exit in time")
             if isinstance(thread_or_process, multiprocessing.Process):