Skip to content

Commit

Permalink
Generate reference docs from proto files (#982)
Browse files Browse the repository at this point in the history
  • Loading branch information
markmandel authored Aug 2, 2024
1 parent 0f66af6 commit f7bee15
Show file tree
Hide file tree
Showing 14 changed files with 1,054 additions and 60 deletions.
2 changes: 1 addition & 1 deletion build/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ build: binary-archive build-image
# Build all binaries
build-all-binaries: ensure-build-image build-linux-binary build-macos-binary build-windows-binary

# Checks for changes to protobuf definitions and generates rust code if there is any.
# Checks for changes to protobuf definitions and generates rust code and reference documentation if there is any.
gen-protobuf: ensure-build-image
docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) \
--entrypoint=cargo $(BUILD_IMAGE_TAG) run -p proto-gen -- generate
Expand Down
8 changes: 5 additions & 3 deletions build/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,16 @@ To build a production release, run:

##### Generating Protobuf Files

If you are adding or editing the `.proto` files in this repository, they will need to be regenerated. To do this,
you will need [protoc](https://grpc.io/docs/protoc-installation/) installed locally.
If you are adding or editing the `.proto` files in this repository, they will need to be regenerated along with the
reference documentation. To do this,
you will need [protoc](https://grpc.io/docs/protoc-installation/) and
[protoc-gen-doc](https://github.com/pseudomuto/protoc-gen-doc) installed locally.

Then run:

`cargo run -p proto-gen -- generate`

Which will identify changes in protobuf definitions and regenerate code appropriately.
Which will identify changes in protobuf definitions and regenerate code and documentation appropriately.

#### Testing

Expand Down
8 changes: 8 additions & 0 deletions build/build-image/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,13 @@ RUN set -eux && \
npm install -g browser-sync && \
echo "source /etc/bash_completion" >> /root/.bashrc

# install protoc-gen-doc
RUN cd /tmp && \
wget --quiet -O protoc-gen-doc.tar.gz https://github.com/pseudomuto/protoc-gen-doc/releases/download/v1.5.1/protoc-gen-doc_1.5.1_linux_amd64.tar.gz && \
tar -xf protoc-gen-doc.tar.gz && \
mv ./protoc-gen-doc /usr/local/bin/ && \
rm protoc-gen-doc.tar.gz

# install gcloud
# Credit: https://cloud.google.com/sdk/docs/install#deb
RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && apt-get update -y && \
Expand Down Expand Up @@ -75,6 +82,7 @@ RUN set -eux; \
cargo install --locked cargo-about && \
cargo install --locked mdbook-variables && \
cargo install --locked cargo-deny && \
cargo install --locked proto-gen && \
rustup --version && \
cargo --version && \
rustc --version
16 changes: 16 additions & 0 deletions crates/proto-gen/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
#
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

[package]
name = "proto-gen"
version = "0.1.0"
Expand Down
62 changes: 52 additions & 10 deletions crates/proto-gen/gen.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,15 +91,13 @@ fn install() {
if !child.wait().expect("tar is not installed").success() {
panic!("failed to extract proto-gen binary from tarball");
}
} else {
if !Command::new("cargo")
.args(["install", "--locked", "-f", "proto-gen"])
.status()
.expect("cargo not installed")
.success()
{
panic!("failed to install proto-gen via cargo");
}
} else if !Command::new("cargo")
.args(["install", "--locked", "-f", "proto-gen"])
.status()
.expect("cargo not installed")
.success()
{
panic!("failed to install proto-gen via cargo");
}
}

Expand Down Expand Up @@ -226,6 +224,50 @@ fn execute(which: &str) {
if !cmd.status().expect("proto-gen was not installed").success() {
panic!("proto-gen {which} failed");
}

if which == "generate" {
docs(files);
}
}

fn docs(files: &[(&str, &[&str])]) {
let mut cmd = Command::new("protoc");

let quilkin_protos: Vec<&(&str, &[&str])> = files
.iter()
.filter(|item| item.0 == "proto/quilkin")
.collect();
let includes: Vec<&(&str, &[&str])> = files
.iter()
.filter(|item| item.0 != "proto/quilkin")
.collect();

for (dir, files) in includes {
if files.is_empty() {
cmd.args(["-I", dir]);
} else {
for file in *files {
cmd.args(["-I".into(), format!("{dir}/{file}.proto")]);
}
}
}

cmd.args(["--doc_out", "./docs/src/services/xds/proto"]);
cmd.args(["--doc_opt", "markdown,index.md"]);

for (dir, files) in quilkin_protos {
for file in *files {
cmd.arg(format!("{dir}/{file}.proto"));
}
}

if !cmd
.status()
.expect("protoc-gen-doc was not installed")
.success()
{
panic!("protoc-gen-doc failed");
}
}

fn copy() {
Expand Down Expand Up @@ -294,7 +336,7 @@ fn copy() {
];

let args: Vec<_> = std::env::args().skip(2).collect();
let name = args.get(0).expect("must provide source name");
let name = args.first().expect("must provide source name");
let path = args.get(1).expect("must provide path");

let Some(ri) = REPOS.iter().find(|r| r.name == name) else {
Expand Down
16 changes: 16 additions & 0 deletions crates/quilkin-proto/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
#
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

[package]
name = "quilkin-proto"
version = "0.1.0"
Expand Down
14 changes: 6 additions & 8 deletions crates/quilkin-proto/src/generated/quilkin/relay/v1alpha1.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,10 +95,8 @@ pub mod aggregated_control_plane_discovery_service_client {
self.inner = self.inner.max_encoding_message_size(limit);
self
}
/// The RPC protocol begins with a single empty DiscoveryResponse
/// initiated by the management server, after that this behaves
/// the same as the management server xDS protocol, except with
/// DiscoveryRequests initiated by the server rather than the client.
/// This RPC is not supported but remains here as part of the xDS standard,
/// as Quilkin only uses a delta configuration transport for performance reasons.
pub async fn stream_aggregated_resources(
&mut self,
request: impl tonic::IntoStreamingRequest<
Expand Down Expand Up @@ -129,6 +127,7 @@ pub mod aggregated_control_plane_discovery_service_client {
));
self.inner.streaming(req, path, codec).await
}
/// Delta (Incremental) xDS implementation
pub async fn delta_aggregated_resources(
&mut self,
request: impl tonic::IntoStreamingRequest<
Expand Down Expand Up @@ -176,10 +175,8 @@ pub mod aggregated_control_plane_discovery_service_server {
>,
> + Send
+ 'static;
/// The RPC protocol begins with a single empty DiscoveryResponse
/// initiated by the management server, after that this behaves
/// the same as the management server xDS protocol, except with
/// DiscoveryRequests initiated by the server rather than the client.
/// This RPC is not supported but remains here as part of the xDS standard,
/// as Quilkin only uses a delta configuration transport for performance reasons.
async fn stream_aggregated_resources(
&self,
request: tonic::Request<
Expand All @@ -200,6 +197,7 @@ pub mod aggregated_control_plane_discovery_service_server {
>
+ Send
+ 'static;
/// Delta (Incremental) xDS implementation
async fn delta_aggregated_resources(
&self,
request: tonic::Request<
Expand Down
16 changes: 16 additions & 0 deletions crates/test/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
#
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

[package]
name = "qt"
version = "0.1.0"
Expand Down
1 change: 1 addition & 0 deletions docs/htmltest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ IgnoreInternalURLs:
- "../../print.html"
IgnoreDirs:
- "api"
- "book/services/xds/proto/"
HTTPHeaders:
User-Agent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
Accept: "text/html"
2 changes: 1 addition & 1 deletion docs/src/SUMMARY.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
- [Providers]()
- [Agones](./services/xds/providers/agones.md)
- [Filesystem](./services/xds/providers/filesystem.md)

- [Protobuf Reference](./services/xds/proto/index.md)
---

- [Relay](./services/relay.md)
Expand Down
53 changes: 20 additions & 33 deletions docs/src/services/xds.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,34 +19,29 @@ $ quilkin manage --help

## Overview

In addition to static configuration provided upon startup, a Quiklin proxy's configuration can also be updated at runtime. The proxy can be configured on startup to talk to a set of management servers which provide it with updates throughout its lifecycle.
In addition to static configuration provided upon startup, a Quiklin proxy's configuration can also be updated at
runtime. The proxy can be configured on startup to talk to a set of management servers which provide it with updates
throughout its lifecycle.

Communication between the proxy and management server uses the [xDS gRPC protocol][xDS], similar to an [envoy proxy]. xDS is one of the standard configuration mechanisms for software proxies and as a result, Quilkin can be setup to discover configuration resources from any API compatible server. Also, given that the protocol is [well specified][xDS-protocol], it is similarly straight-forward to implement a custom server to suit any deployment's needs.
Communication between the proxy and management server uses the [xDS gRPC protocol][xDS], similar to an [envoy proxy].
xDS is one of the standard configuration mechanisms for software proxies and as a result, Quilkin can be setup to
discover configuration resources from any API compatible server. Also, given that the protocol
is [well specified][xDS-protocol], it is similarly straight-forward to implement a custom server to suit any
deployment's needs.

As described within the [xDS-api] documentation, the xDS API comprises a set of resource discovery APIs, each serving a specific set of configuration resource types, while the protocol itself comes in several [variants][xds-variants].
Quilkin implements the **Aggregated Discovery Service (ADS)** _State of the World (SotW)_ variant with gRPC.
As described within the [xDS-api] documentation, the xDS API comprises a set of resource discovery APIs, each serving a
specific set of configuration resource types, while the protocol itself comes in several [variants][xds-variants].
Quilkin implements the **Aggregated Discovery Service (ADS)** _[Incremental xDS][incremental-xds]_ variant with gRPC.

## Supported APIs
## xDS Configuration Resources

Since the range of resources configurable by the xDS API extends that of Quilkin's domain (i.e being UDP based, Quilkin does not have a need for HTTP/TCP resources), only a subset of the API is supported. The following lists these relevant parts and any limitation to the provided support as a result:
Heavily influenced by xDS's [Cluster Discovery Service][CDS], [Endpoint Discovery Service][EDS], and
[Listener Discovery Service][LDS], Quilkin utilises its own custom Configuration xDS resources
[(reference)][config.proto] to enable dynamic configuration of Datacenters, Clusters, Endpoints and Filters through
the Aggregated Discovery Service.

- **Cluster Discovery Service [(CDS)][CDS]**: Provides information about known clusters and their membership information.
* The proxy uses these resources to discover clusters and their endpoints.
* While cluster topology information like [locality] can be provided in the configuration, the proxy currently does not use this information (support may be included in the future however).
* Any [load balancing information][lbpolicy] included in this resource is ignored. For load balancing, use [Quilkin filters][filters-doc] instead.
* Only [cluster discovery type] `STATIC` and `EDS` is supported. Configuration including other discovery types e.g `LOGICAL_DNS` is rejected.

- **Endpoint Discovery Service [(EDS)][EDS]**: Provides information about endpoints.
* The proxy uses these resources to discover information about endpoints like their IP addresses.
* Endpoints may provide [Endpoint Metadata][endpoint-metadata] via the [metadata][xds-endpoint-metadata] field. These metadata will be visible to filters as part of the corresponding endpoints information when processing packets.
* Only [socket addresses] are supported on an endpoint's address configuration - i.e an IP address and port number combination. Configuration including any other type of addressing e.g named pipes will be rejected.
* Any [load balancing information][clapolicy] included in this resource is ignored. For load balancing, use [Quilkin filters][filters-doc] instead.

- **Listener Discovery Service [(LDS)][LDS]**: Provides information about [Filters and Filter Chains][filters-doc].
* Only the `name` and `filter_chains` fields in the [Listener resource][listener-resource] are used by the proxy. The rest are ignored.
* Since Quilkin only uses one filter chain per proxy, at most one filter chain can be provided in the resource. Otherwise the configuration is rejected.
* Only the list of [filters][xds-filters] specified in the [filter chain][xds-filter-chain] is used by the proxy - i.e other fields like `filter_chain_match` are ignored. This list also specifies the order that the corresponding filter chain will be constructed.
* gRPC proto configuration for Quilkin's built-in filters [can be found here][filter-protos]. They are equivalent to the filter's static configuration.
This [above reference][config.proto] also provides documentation for all the ProtoBuf definitions for all
[Filters][filters-doc] that are available in Quilkin.

## Connecting to an xDS management server

Expand All @@ -55,25 +50,17 @@ the `management_servers` [command line](../../api/quilkin/struct.Proxy.html#stru


[xDS]: https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#xds-rest-and-grpc-protocol
[incremental-xds]: https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#incremental-xds
[envoy proxy]: https://www.envoyproxy.io/docs/envoy/latest/
[xDS-protocol]: https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#the-xds-transport-protocol
[xDS-api]: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/operations/dynamic_configuration
[config.proto]: ./xds/proto/index.md#quilkin_config_v1alpha1_config-proto
[CDS]: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/operations/dynamic_configuration#cds
[EDS]: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/operations/dynamic_configuration#eds
[LDS]: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/operations/dynamic_configuration#lds
[cluster discovery type]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#enum-config-cluster-v3-cluster-discoverytype
[lbpolicy]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#enum-config-cluster-v3-cluster-lbpolicy
[clapolicy]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/endpoint/v3/endpoint.proto#config-endpoint-v3-clusterloadassignment-policy
[locality]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/base.proto#config-core-v3-locality
[socket addresses]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#config-core-v3-address
[filters-doc]: ./proxy/filters.md
[listener-resource]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/listener/v3/listener.proto#config-listener-v3-listener
[xds-filters]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/listener/v3/listener_components.proto#envoy-v3-api-msg-config-listener-v3-filter
[xds-filter-chain]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/listener/v3/listener_components.proto#config-listener-v3-filterchain
[xds-variants]: https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#variants-of-the-xds-transport-protocol
[filter-protos]: https://github.com/googleforgames/quilkin/tree/{{GITHUB_REF_NAME}}/proto/quilkin/filters
[xds-endpoint-metadata]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/base.proto#envoy-v3-api-msg-config-core-v3-metadata
[endpoint-metadata]: proxy.md#endpoint-metadata
[control-plane]: https://github.com/googleforgames/quilkin/tree/{{GITHUB_REF_NAME}}/xds
[Kubernetes]: https://kubernetes.io/

Loading

0 comments on commit f7bee15

Please sign in to comment.