From 6efb1e84b361a89add3280d08d94f335dba22bfd Mon Sep 17 00:00:00 2001 From: "Dr. Dennis Wittich" Date: Wed, 13 Nov 2024 13:33:02 +0100 Subject: [PATCH 1/4] fix custom creation date --- learning_loop_node/data_classes/image_metadata.py | 2 +- learning_loop_node/detector/detector_node.py | 5 +++-- learning_loop_node/detector/outbox.py | 13 ++++++++----- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/learning_loop_node/data_classes/image_metadata.py b/learning_loop_node/data_classes/image_metadata.py index 98496a2..af95d00 100644 --- a/learning_loop_node/data_classes/image_metadata.py +++ b/learning_loop_node/data_classes/image_metadata.py @@ -28,7 +28,7 @@ class ImageMetadata(): tags: List[str] = field(default_factory=list, metadata={ 'description': 'List of tags'}) - date: Optional[str] = field(default_factory=current_datetime, metadata={ + created: Optional[str] = field(default_factory=current_datetime, metadata={ 'description': 'Creation date of the image'}) source: Optional[str] = field(default=None, metadata={ 'description': 'Source of the image'}) diff --git a/learning_loop_node/detector/detector_node.py b/learning_loop_node/detector/detector_node.py index 5160425..07252ac 100644 --- a/learning_loop_node/detector/detector_node.py +++ b/learning_loop_node/detector/detector_node.py @@ -140,7 +140,6 @@ def setup_sio_server(self) -> None: @self.sio.event async def detect(sid, data: Dict) -> Dict: - self.log.debug('running detect via socketio') try: np_image = np.frombuffer(data['image'], np.uint8) det = await self.get_detections( @@ -153,7 +152,6 @@ async def detect(sid, data: Dict) -> Dict: if det is None: return {'error': 'no model loaded'} detection_dict = jsonable_encoder(asdict(det)) - self.log.debug('detect via socketio finished') return detection_dict except Exception as e: self.log.exception('could not detect via socketio') @@ -188,6 +186,9 @@ async def upload(sid, data: Dict) -> Optional[Dict]: source = data.get('source', None) creation_date = data.get('creation_date', None) + self.log.debug('running upload via socketio. tags: %s, source: %s, creation_date: %s', + tags, source, creation_date) + loop = asyncio.get_event_loop() try: await loop.run_in_executor(None, self.outbox.save, data['image'], image_metadata, tags, source, creation_date) diff --git a/learning_loop_node/detector/outbox.py b/learning_loop_node/detector/outbox.py index c29cc45..224c475 100644 --- a/learning_loop_node/detector/outbox.py +++ b/learning_loop_node/detector/outbox.py @@ -76,10 +76,11 @@ def save(self, return tmp = f'{GLOBALS.data_folder}/tmp/{identifier}' image_metadata.tags = tags - if creation_date and self._is_valid_isoformat(creation_date): - image_metadata.date = creation_date + if self._is_valid_isoformat(creation_date): + image_metadata.created = creation_date else: - image_metadata.date = identifier + image_metadata.created = identifier + image_metadata.source = source or 'unknown' os.makedirs(tmp, exist_ok=True) @@ -94,7 +95,9 @@ def save(self, else: self.log.error('Could not rename %s to %s', tmp, self.path + '/' + identifier) - def _is_valid_isoformat(self, date: str) -> bool: + def _is_valid_isoformat(self, date: Optional[str]) -> bool: + if date is None: + return True try: datetime.fromisoformat(date) return True @@ -153,7 +156,7 @@ async def _upload_batch(self, items: List[str]): self.log.exception('Could not upload images') return finally: - self.log.info('Closing files') + self.log.debug('Closing files') for _, file in data: file.close() From 39c4dd9385df6b130f2ea88476fe0f314f3cfa77 Mon Sep 17 00:00:00 2001 From: Niklas Neugebauer Date: Wed, 13 Nov 2024 14:11:59 +0100 Subject: [PATCH 2/4] move None check back outside _is_valid_isoformat --- learning_loop_node/detector/outbox.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/learning_loop_node/detector/outbox.py b/learning_loop_node/detector/outbox.py index 224c475..93bdb8c 100644 --- a/learning_loop_node/detector/outbox.py +++ b/learning_loop_node/detector/outbox.py @@ -76,7 +76,7 @@ def save(self, return tmp = f'{GLOBALS.data_folder}/tmp/{identifier}' image_metadata.tags = tags - if self._is_valid_isoformat(creation_date): + if creation_date is None or self._is_valid_isoformat(creation_date): image_metadata.created = creation_date else: image_metadata.created = identifier @@ -95,9 +95,7 @@ def save(self, else: self.log.error('Could not rename %s to %s', tmp, self.path + '/' + identifier) - def _is_valid_isoformat(self, date: Optional[str]) -> bool: - if date is None: - return True + def _is_valid_isoformat(self, date: str) -> bool: try: datetime.fromisoformat(date) return True From 232c01bda30dbb4296356038ab14910e0153b195 Mon Sep 17 00:00:00 2001 From: Niklas Neugebauer Date: Wed, 13 Nov 2024 16:27:30 +0100 Subject: [PATCH 3/4] fix: revert back to using identifier as creation data if None is provided --- learning_loop_node/detector/outbox.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/learning_loop_node/detector/outbox.py b/learning_loop_node/detector/outbox.py index 93bdb8c..201b245 100644 --- a/learning_loop_node/detector/outbox.py +++ b/learning_loop_node/detector/outbox.py @@ -76,7 +76,7 @@ def save(self, return tmp = f'{GLOBALS.data_folder}/tmp/{identifier}' image_metadata.tags = tags - if creation_date is None or self._is_valid_isoformat(creation_date): + if self._is_valid_isoformat(creation_date): image_metadata.created = creation_date else: image_metadata.created = identifier @@ -95,7 +95,9 @@ def save(self, else: self.log.error('Could not rename %s to %s', tmp, self.path + '/' + identifier) - def _is_valid_isoformat(self, date: str) -> bool: + def _is_valid_isoformat(self, date: Optional[str]) -> bool: + if date is None: + return False try: datetime.fromisoformat(date) return True From 75e7e07928252e13844ab0b004c29735d31f95ab Mon Sep 17 00:00:00 2001 From: "Dr. Dennis Wittich" Date: Wed, 13 Nov 2024 16:39:22 +0100 Subject: [PATCH 4/4] fix test --- learning_loop_node/loop_communication.py | 6 +++--- mock_detector/app_code/tests/test_detector.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/learning_loop_node/loop_communication.py b/learning_loop_node/loop_communication.py index 13505e0..7f44cb2 100644 --- a/learning_loop_node/loop_communication.py +++ b/learning_loop_node/loop_communication.py @@ -35,7 +35,7 @@ def __init__(self) -> None: else: self.async_client = httpx.AsyncClient(base_url=self.base_url, timeout=Timeout(60.0)) - logging.info(f'Loop interface initialized with base_url: {self.base_url} / user: {self.username}') + logging.info('Loop interface initialized with base_url: %s / user: %s', self.base_url, self.username) def websocket_url(self) -> str: return f'ws{"s" if "learning-loop.ai" in self.host else ""}://' + self.host @@ -48,7 +48,7 @@ async def ensure_login(self, relogin=False) -> None: self.async_client.cookies.clear() response = await self.async_client.post('/api/login', data={'username': self.username, 'password': self.password}) if response.status_code != 200: - logging.info(f'Login failed with response: {response}') + logging.info('Login failed with response: %s', response) raise LoopCommunicationException('Login failed with response: ' + str(response)) self.async_client.cookies.update(response.cookies) @@ -57,7 +57,7 @@ async def logout(self) -> None: response = await self.async_client.post('/api/logout') if response.status_code != 200: - logging.info(f'Logout failed with response: {response}') + logging.info('Logout failed with response: %s', response) raise LoopCommunicationException('Logout failed with response: ' + str(response)) self.async_client.cookies.clear() diff --git a/mock_detector/app_code/tests/test_detector.py b/mock_detector/app_code/tests/test_detector.py index a10c8ef..19719fc 100644 --- a/mock_detector/app_code/tests/test_detector.py +++ b/mock_detector/app_code/tests/test_detector.py @@ -38,4 +38,4 @@ async def test_sio_detect(test_detector_node: DetectorNode, sio): assert response['point_detections'] == [] assert response['segmentation_detections'] == [] assert response['tags'] == [] - assert 'date' in response + assert 'created' in response