Syncing recent changes. (#1124)

* Syncing recent changes.
* Update client_build.py: fix the config check.
* Add executable bit to the grr_fleetspeak_client.sh.
* Expose mysql port 3306 in Docker compose config to ensure e2e tests complete.

.dockerignore

@@ -1,34 +1,31 @@
-# Unfortunately the ** syntax isn't supported before docker 1.10
-# https://github.com/docker/docker/issues/13113
-# and installing a newer version is difficult.
+.dockerignore
+.git/
+.github/
+gha-creds-*.json
 *.changes
-*.deb
-*.dsc
-*.egg-info
-*/*/server.local.yaml
-*.pyc
-*/*.pyc
-*/*/*.pyc
-*/*/*/*.pyc
-*/*/*/*/*.pyc
-*_pb2.*
-*/*_pb2.*
-*/*/*_pb2.*
-.git
+**/*.deb
+**/*.dsc
+**/*.egg-info/
+**/*.pyc
+**/*_pb2.*
+**/*.o
+**/*.so
+**/*.whl
+**/__pycache__/
+**/build/
+**/dist/
+**/server.local.yaml
 ACKNOWLEDGEMENTS
 AUTHORS
-build
-dist
-executables
+executables/
 !executables/windows/templates/unzipsfx/*.exe
-gha-creds-*.json
-grr/config/grr_response_templates
-grr/gui/static/bower_components
-grr/gui/static/node_modules
-grr/gui/static/tmp
-grr/var
+grr/config/grr_response_templates/
+grr/gui/static/bower_components/
+grr/gui/static/node_modules/
+grr/gui/static/tmp/
+grr/var/
 grr_server*.tar.gz
 LICENSE
 README.md
-travis
-vagrant
+travis/
+vagrant/

.gitignore

@@ -1,16 +1,27 @@
+*.deb
+*.dsc
+*.log
+*.orig
+*.rej
 *.pyc
-*_pb2.py
+*_pb2.*
 *.swp
 *.o
 *.so
 *.pb.cc
 *.pb.h
+*.whl
 *~
+
+# Ignore generated credentials from google-github-actions/auth
+gha-creds-*.json
+
+__pycache__/
 .idea/
 build/
 dist/
 artifacts/*.yaml
-grr.egg-info/
+gcs_upload_dir/
 get-pip.py
 grr/core/grr_response_core/artifacts/*.yaml
 *.egg-info/
@@ -20,7 +31,5 @@ tmp/
 grr/server/grr_response_server/gui/static/fonts/
 installers/
 GRRlog.txt
-*.log
-grr/server/grr_response_server/gui/static/third-party
-grr/server/grr_response_server/gui/ui/.angular
-gha-creds-*.json
+grr/server/grr_response_server/gui/static/third-party/
+grr/server/grr_response_server/gui/ui/.angular/

CHANGELOG.md

@@ -17,6 +17,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Removed the `ListParsedFlowResults` API method.
 * Removed support for the `GREP` artifact source (these were internal to GRR and
   not part of the [official specification](https://artifacts.readthedocs.io/en/latest/sources/Format-specification.html).
+* Removed `ApiClient.users` field which contained duplicate information present in `ApiClient.knowledge_base.users` field.
+* Removed no longer used or relevant configuration options:
+  * `Server.fleetspeak_enabled`
+  * `Client.fleetspeak_enabled`
+  * `Client.server_urls`
+  * `Client.control_urls`
+  * `Client.server_serial_number`
+  * `Client.poll_min`
+  * `Client.poll_max`
+  * `Client.error_poll_min`
+  * `Client.labels`
+  * `Client.rss_max`
+  * `Client.rss_max_hard`
+  * `Frontend.bind_port`
+  * `Frontend.port_max`
+  * `Nanny.child_binary`
+  * `Nanny.child_command_line`
+  * `Nanny.service_name`
+  * `Nanny.service_description`
+  * `Nanny.statusfile`
+  * `Nanny.binary`
+  * `Nanny.service_binary_name`
+
+### Changed
+
+* Renamed `restricted_flow_users` and `restricted_flow_groups` to `admin_users`
+  and `admin_groups`. The new entries can be used to grant access for
+  Client/Hunt/CronJob approvals as well as restricted flows.
 
 ## [3.4.7.4] - 2024-05-28
 

Dockerfile

@@ -6,15 +6,28 @@
 # time a new a PUSH happens in the GRR github repository.
 #
 # Examples:
-# - Run a grr server component (e.g. admin_ui):
+# - Run a GRR server component (e.g. admin_ui):
 #
 #   $ docker run -it \
 #       -v $(pwd)/docker_config_files/server:/configs \
 #       ghcr.io/google/grr:latest \
 #       "-component" "admin_ui" \
 #       "-config" "/configs/grr.server.yaml"
 #
-# - Run the grr client component:
+# - Run the GRR client component via repacking client templates:
+#   Client installers for different operating systems are created by
+#   repacking client templates, which are included in the GRR Docker image
+#   downloaded from github (they are currently only build in the github workflow
+#   that creates the GRR Docker image and not in a local build). The Fleetspeak
+#   client requires connectivity to the Fleetspeak server, we recommend running
+#   this client in the Docker Compose stack (see compose.yaml for details)
+#   otherwise the config needs to be adjusted. The Docker Compose stack is also
+#   taking care of repacking the client templates and installing them in the
+#   GRR client container.
+#
+#   If you nontheless want to repack the client templates, install them in
+#   a local container and start the client, you can use the following commands:
+#
 #   -- Start the container and mount the client config directory:
 #       $ docker run -it \
 #          -v $(pwd)/docker_config_files/client:/configs \
@@ -26,7 +39,7 @@
 #      resulting debian file inside the container:
 #       root@<CONTAINER ID> $ /configs/repack_install_client.sh
 #
-#   -- Start fleetspeak and grr clients:
+#   -- Start Fleetspeak and GRR clients:
 #       root@<CONTAINER ID> $ fleetspeak-client -config /configs/client.config
 #
 #   -- (Optional) To verify if the client runs, check if the two expected
@@ -35,6 +48,8 @@
 #             ...        COMMAND
 #             ...        fleetspeak-client -config /configs/client.config
 #             ...        python -m grr_response_client.client ...
+# - To run a GRR client container without repacking checkout out the
+#   Dockerfile.client file.
 
 FROM ubuntu:22.04
 

api_client/python/grr_api_client/api.py

@@ -21,7 +21,9 @@
 from grr_api_client import yara
 from grr_response_proto import flows_pb2
 from grr_response_proto.api import config_pb2
-from grr_response_proto.api import hunt_pb2
+# TODO: Remove this import once the SignedCommands API is
+# implemented. Currently required to be able to parse responses from the server.
+from grr_response_proto.api import signed_commands_pb2  # pylint: disable=unused-import
 
 
 class GrrApi(object):
@@ -62,14 +64,6 @@ def CreateHunt(
         context=self._context,
     )
 
-  def CreatePerClientFileCollectionHunt(
-      self,
-      hunt_args: hunt_pb2.ApiCreatePerClientFileCollectionHuntArgs,
-  ) -> hunt.Hunt:
-    return hunt.CreatePerClientFileCollectionHunt(
-        hunt_args, context=self._context
-    )
-
   def ListHunts(self) -> utils.ItemsIterator[hunt.Hunt]:
     return hunt.ListHunts(context=self._context)
 
@@ -82,6 +76,23 @@ def ListGrrBinaries(self) -> utils.ItemsIterator[config.GrrBinary]:
   def ListArtifacts(self) -> utils.ItemsIterator[artifact.Artifact]:
     return artifact.ListArtifacts(context=self._context)
 
+  def UploadArtifact(self, yaml: str) -> None:
+    # pylint: disable=line-too-long
+    # fmt: off
+    """Uploads the given [YAML artifact definition][1] to the GRR server.
+
+    [1]: https://artifacts.readthedocs.io/en/latest/sources/Format-specification.html
+
+    Args:
+      yaml: YAML with the artifact definition.
+
+    Returns:
+      Nothing.
+    """
+    # pylint: enable=line-too-long
+    # fmt: on
+    return artifact.UploadArtifact(context=self._context, yaml=yaml)
+
   def GrrBinary(
       self,
       binary_type: config_pb2.ApiGrrBinary.Type,

api_client/python/grr_api_client/artifact.py

@@ -29,3 +29,28 @@ def ListArtifacts(
   return utils.MapItemsIterator(
       lambda data: Artifact(data=data, context=context), items
   )
+
+
+def UploadArtifact(
+    context: api_context.GrrApiContext,
+    yaml: str,
+) -> None:
+  # pylint: disable=line-too-long
+  # fmt: off
+  """Uploads the given [YAML artifact definition][1] to the GRR server.
+
+  [1]: https://artifacts.readthedocs.io/en/latest/sources/Format-specification.html
+
+  Args:
+    context: GRR API context to use.
+    yaml: YAML with the artifact definition.
+
+  Returns:
+    Nothing.
+  """
+  # pylint: enable=line-too-long
+  # fmt: on
+  args = api_artifact_pb2.ApiUploadArtifactArgs()
+  args.artifact = yaml
+
+  context.SendRequest("UploadArtifact", args)

api_client/python/grr_api_client/connectors/http.py

@@ -1,23 +1,16 @@
 #!/usr/bin/env python
 """HTTP API connector implementation."""
 
+from collections.abc import Iterable, Iterator
 import contextlib
 import json
 import logging
 import re
-from typing import Any
-from typing import Dict
-from typing import Iterable
-from typing import Iterator
-from typing import NamedTuple
-from typing import Optional
-from typing import Tuple
-from typing import Union
+from typing import Any, NamedTuple, Optional, Union
 from urllib import parse as urlparse
 
 import pkg_resources
 import requests
-
 from werkzeug import routing
 
 from google.protobuf import descriptor
@@ -122,8 +115,8 @@ class HttpConnector(abstract.Connector):
   def __init__(
       self,
       api_endpoint: str,
-      auth: Optional[Tuple[str, str]] = None,
-      proxies: Optional[Dict[str, str]] = None,
+      auth: Optional[tuple[str, str]] = None,
+      proxies: Optional[dict[str, str]] = None,
       verify: Optional[bool] = None,
       cert: Optional[str] = None,
       trust_env: Optional[bool] = None,
@@ -145,7 +138,7 @@ def __init__(
       validate_version = True
 
     self.api_endpoint: str = api_endpoint
-    self.proxies: Optional[Dict[str, str]] = proxies
+    self.proxies: Optional[dict[str, str]] = proxies
     self.verify: bool = verify
     self.cert: Optional[str] = cert
     self._page_size: int = page_size
@@ -157,7 +150,7 @@ def __init__(
     self.session.verify = verify
 
     self.csrf_token: Optional[str] = None
-    self.api_methods: Dict[str, reflection_pb2.ApiMethod] = {}
+    self.api_methods: dict[str, reflection_pb2.ApiMethod] = {}
 
     self._server_version: Optional[VersionTuple] = None
     self._api_client_version: Optional[VersionTuple] = None
@@ -281,7 +274,7 @@ def _GetMethodUrlAndPathParamsNames(
       self,
       handler_name: str,
       args: message.Message,
-  ) -> Tuple[reflection_pb2.ApiMethod, str, Iterable[str]]:
+  ) -> tuple[reflection_pb2.ApiMethod, str, Iterable[str]]:
     path_params = {}  # Dict[str, Union[int, str]]
     if args:
       for field, value in args.ListFields():
@@ -305,7 +298,7 @@ def _ArgsToQueryParams(
       self,
       args: Optional[message.Message],
       exclude_names: Iterable[str],
-  ) -> Dict[str, Union[int, str]]:
+  ) -> dict[str, Union[int, str]]:
     if args is None:
       return {}
 

api_client/python/grr_api_client/flow.py

@@ -98,9 +98,14 @@ def ListLogs(self) -> utils.ItemsIterator[FlowLog]:
     items = self._context.SendIteratorRequest("ListFlowLogs", args)
     return utils.MapItemsIterator(lambda data: FlowLog(data=data), items)
 
-  def GetFilesArchive(self) -> utils.BinaryChunkIterator:
+  def GetFilesArchive(
+      self,
+      archive_format: flow_pb2.ApiGetFlowFilesArchiveArgs.ArchiveFormat = flow_pb2.ApiGetFlowFilesArchiveArgs.ArchiveFormat.ZIP,
+  ) -> utils.BinaryChunkIterator:
     args = flow_pb2.ApiGetFlowFilesArchiveArgs(
-        client_id=self.client_id, flow_id=self.flow_id
+        client_id=self.client_id,
+        flow_id=self.flow_id,
+        archive_format=archive_format,
     )
     return self._context.SendStreamingRequest("GetFlowFilesArchive", args)
 

api_client/python/grr_api_client/hunt.py

@@ -461,19 +461,6 @@ def CreateHunt(
   return Hunt(data=data, context=context)
 
 
-def CreatePerClientFileCollectionHunt(
-    hunt_args: hunt_pb2.ApiCreatePerClientFileCollectionHuntArgs,
-    context: context_lib.GrrApiContext,
-) -> Hunt:
-  """Createt a per-client file collection hunt."""
-
-  data = context.SendRequest("CreatePerClientFileCollectionHunt", hunt_args)
-  if not isinstance(data, hunt_pb2.ApiHunt):
-    raise TypeError(f"Unexpected response type: '{type(data)}'")
-
-  return Hunt(data=data, context=context)
-
-
 def ListHunts(context: context_lib.GrrApiContext) -> utils.ItemsIterator[Hunt]:
   """List all GRR hunts."""
 

api_client/python/grr_api_client/root.py

@@ -13,6 +13,7 @@
 from grr_api_client import context as api_context
 from grr_api_client import utils
 from grr_response_proto.api import config_pb2
+from grr_response_proto.api import signed_commands_pb2
 from grr_response_proto.api import user_pb2
 from grr_response_proto.api.root import binary_management_pb2
 from grr_response_proto.api.root import user_management_pb2
@@ -228,3 +229,17 @@ def GrrBinary(
     return GrrBinaryRef(
         binary_type=binary_type, path=path, context=self._context
     )
+
+  def CreateSignedCommands(
+      self,
+      commands: signed_commands_pb2.ApiSignedCommands,
+  ) -> None:
+    """Creates a command signer."""
+    args = signed_commands_pb2.ApiCreateSignedCommandsArgs()
+    args.signed_commands.extend(commands.signed_commands)
+
+    self._context.SendRequest("CreateSignedCommands", args)
+
+  def DeleteAllSignedCommands(self):
+    """Deletes all signed commands."""
+    self._context.SendRequest("DeleteAllSignedCommands", args=None)