Merge branch 'main' into spartan0x117/add-flow-by-example-chapter-0

CHANGELOG.md

@@ -32,6 +32,9 @@ Main (unreleased)
 
 - `pyroscope.ebpf` support python on arm64 platforms. (@korniltsev)
 
+- Added links between compatible components in the documentation to make it
+  easier to discover them. (@thampiotr)
+
 ### Bugfixes
 
 - Update `pyroscope.ebpf` to fix a logical bug causing to profile to many kthreads instead of regular processes https://github.com/grafana/pyroscope/pull/2778 (@korniltsev)

Makefile

@@ -287,7 +287,7 @@ smoke-image:
 #
 
 .PHONY: generate generate-crds generate-drone generate-helm-docs generate-helm-tests generate-manifests generate-dashboards generate-protos generate-ui generate-versioned-files
-generate: generate-crds generate-drone generate-helm-docs generate-helm-tests generate-manifests generate-dashboards generate-protos generate-ui generate-versioned-files
+generate: generate-crds generate-drone generate-helm-docs generate-helm-tests generate-manifests generate-dashboards generate-protos generate-ui generate-versioned-files generate-docs
 
 generate-crds:
 ifeq ($(USE_CONTAINER),1)
@@ -350,6 +350,12 @@ else
 	sh ./tools/gen-versioned-files/gen-versioned-files.sh
 endif
 
+generate-docs:
+ifeq ($(USE_CONTAINER),1)
+	$(RERUN_IN_CONTAINER)
+else
+	go generate ./docs
+endif
 #
 # Other targets
 #

component/common/loki/client/internal/marker_file_handler.go

@@ -16,8 +16,10 @@ const (
 	MarkerFolderName = "remote"
 	MarkerFileName   = "segment_marker"
 
-	MarkerFolderMode os.FileMode = 0o700
-	MarkerFileMode   os.FileMode = 0o600
+	MarkerFolderMode        os.FileMode = 0o700
+	MarkerWindowsFolderMode os.FileMode = 0o777
+	MarkerFileMode          os.FileMode = 0o600
+	MarkerWindowsFileMode   os.FileMode = 0o666
 )
 
 // MarkerFileHandler is a file-backed wal.Marker, that also allows one to write to the backing store as particular

component/common/loki/client/internal/marker_file_handler_test.go

@@ -3,6 +3,7 @@ package internal
 import (
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 
 	"github.com/go-kit/log"
@@ -57,10 +58,18 @@ func TestMarkerFileHandler(t *testing.T) {
 		// check folder first
 		stats, err := os.Stat(filepath.Join(dir, MarkerFolderName))
 		require.NoError(t, err)
-		require.Equal(t, MarkerFolderMode, stats.Mode().Perm())
+		if runtime.GOOS == "windows" {
+			require.Equal(t, MarkerWindowsFolderMode, stats.Mode().Perm())
+		} else {
+			require.Equal(t, MarkerFolderMode, stats.Mode().Perm())
+		}
 		// then file
 		stats, err = os.Stat(filepath.Join(dir, MarkerFolderName, MarkerFileName))
 		require.NoError(t, err)
-		require.Equal(t, MarkerFileMode, stats.Mode().Perm())
+		if runtime.GOOS == "windows" {
+			require.Equal(t, MarkerWindowsFileMode, stats.Mode().Perm())
+		} else {
+			require.Equal(t, MarkerFileMode, stats.Mode().Perm())
+		}
 	})
 }

component/metadata/metadata.go

@@ -0,0 +1,193 @@
+package metadata
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/grafana/agent/component"
+	_ "github.com/grafana/agent/component/all"
+	"github.com/grafana/agent/component/common/loki"
+	"github.com/grafana/agent/component/discovery"
+	"github.com/grafana/agent/component/otelcol"
+	"github.com/grafana/agent/component/pyroscope"
+	"github.com/prometheus/prometheus/storage"
+)
+
+//TODO(thampiotr): Instead of metadata package reaching into registry, we'll migrate to using a YAML schema file that
+//				   contains information about all the available components. This file will be generated separately and
+//				   can be used by other tools.
+
+type Type struct {
+	Name string
+	// Returns true if provided args include this type (including nested structs)
+	existsInArgsFn func(args component.Arguments) bool
+	// Returns true if provided exports include this type (including nested structs)
+	existsInExportsFn func(exports component.Exports) bool
+}
+
+func (t Type) String() string {
+	return fmt.Sprintf("Type[%s]", t.Name)
+}
+
+var (
+	TypeTargets = Type{
+		Name: "Targets",
+		existsInArgsFn: func(args component.Arguments) bool {
+			return hasFieldOfType(args, reflect.TypeOf([]discovery.Target{}))
+		},
+		existsInExportsFn: func(exports component.Exports) bool {
+			return hasFieldOfType(exports, reflect.TypeOf([]discovery.Target{}))
+		},
+	}
+
+	TypeLokiLogs = Type{
+		Name: "Loki `LogsReceiver`",
+		existsInArgsFn: func(args component.Arguments) bool {
+			return hasFieldOfType(args, reflect.TypeOf([]loki.LogsReceiver{}))
+		},
+		existsInExportsFn: func(exports component.Exports) bool {
+			return hasFieldOfType(exports, reflect.TypeOf(loki.NewLogsReceiver()))
+		},
+	}
+
+	TypePromMetricsReceiver = Type{
+		Name: "Prometheus `MetricsReceiver`",
+		existsInArgsFn: func(args component.Arguments) bool {
+			return hasFieldOfType(args, reflect.TypeOf([]storage.Appendable{}))
+		},
+		existsInExportsFn: func(exports component.Exports) bool {
+			var a *storage.Appendable = nil
+			return hasFieldOfType(exports, reflect.TypeOf(a).Elem())
+		},
+	}
+
+	TypePyroProfilesReceiver = Type{
+		Name: "Pyroscope `ProfilesReceiver`",
+		existsInArgsFn: func(args component.Arguments) bool {
+			return hasFieldOfType(args, reflect.TypeOf([]pyroscope.Appendable{}))
+		},
+		existsInExportsFn: func(exports component.Exports) bool {
+			var a *pyroscope.Appendable = nil
+			return hasFieldOfType(exports, reflect.TypeOf(a).Elem())
+		},
+	}
+
+	TypeOTELReceiver = Type{
+		Name: "OpenTelemetry `otelcol.Consumer`",
+		existsInArgsFn: func(args component.Arguments) bool {
+			return hasFieldOfType(args, reflect.TypeOf([]otelcol.Consumer{}))
+		},
+		existsInExportsFn: func(exports component.Exports) bool {
+			var a *otelcol.Consumer = nil
+			return hasFieldOfType(exports, reflect.TypeOf(a).Elem())
+		},
+	}
+
+	AllTypes = []Type{
+		TypeTargets,
+		TypeLokiLogs,
+		TypePromMetricsReceiver,
+		TypePyroProfilesReceiver,
+		TypeOTELReceiver,
+	}
+)
+
+type Metadata struct {
+	accepts []Type
+	exports []Type
+}
+
+func (m Metadata) Empty() bool {
+	return len(m.accepts) == 0 && len(m.exports) == 0
+}
+
+func (m Metadata) AllTypesAccepted() []Type {
+	return m.accepts
+}
+
+func (m Metadata) AllTypesExported() []Type {
+	return m.exports
+}
+
+func (m Metadata) AcceptsType(t Type) bool {
+	for _, a := range m.accepts {
+		if a.Name == t.Name {
+			return true
+		}
+	}
+	return false
+}
+
+func (m Metadata) ExportsType(t Type) bool {
+	for _, o := range m.exports {
+		if o.Name == t.Name {
+			return true
+		}
+	}
+	return false
+}
+
+func ForComponent(name string) (Metadata, error) {
+	reg, ok := component.Get(name)
+	if !ok {
+		return Metadata{}, fmt.Errorf("could not find component %q", name)
+	}
+	return inferMetadata(reg.Args, reg.Exports), nil
+}
+
+func inferMetadata(args component.Arguments, exports component.Exports) Metadata {
+	m := Metadata{}
+	for _, t := range AllTypes {
+		if t.existsInArgsFn(args) {
+			m.accepts = append(m.accepts, t)
+		}
+		if t.existsInExportsFn(exports) {
+			m.exports = append(m.exports, t)
+		}
+	}
+	return m
+}
+
+func hasFieldOfType(obj interface{}, fieldType reflect.Type) bool {
+	objValue := reflect.ValueOf(obj)
+
+	// If the object is a pointer, dereference it
+	for objValue.Kind() == reflect.Ptr {
+		objValue = objValue.Elem()
+	}
+
+	// If the object is not a struct or interface, return false
+	if objValue.Kind() != reflect.Struct && objValue.Kind() != reflect.Interface {
+		return false
+	}
+
+	for i := 0; i < objValue.NumField(); i++ {
+		fv := objValue.Field(i)
+		ft := fv.Type()
+
+		// If the field type matches the given type, return true
+		if ft == fieldType {
+			return true
+		}
+
+		if fv.Kind() == reflect.Interface && fieldType.AssignableTo(ft) {
+			return true
+		}
+
+		// If the field is a struct, recursively check its fields
+		if fv.Kind() == reflect.Struct {
+			if hasFieldOfType(fv.Interface(), fieldType) {
+				return true
+			}
+		}
+
+		// If the field is a pointer, create a new instance of the pointer type and recursively check its fields
+		if fv.Kind() == reflect.Ptr {
+			if hasFieldOfType(reflect.New(ft.Elem()).Interface(), fieldType) {
+				return true
+			}
+		}
+	}
+
+	return false
+}

component/metadata/metadata_test.go

@@ -0,0 +1,94 @@
+package metadata
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_inferMetadata(t *testing.T) {
+	tests := []struct {
+		name     string
+		expected Metadata
+	}{
+		{
+			name:     "discovery.dns",
+			expected: Metadata{exports: []Type{TypeTargets}},
+		},
+		{
+			name: "discovery.relabel",
+			expected: Metadata{
+				accepts: []Type{TypeTargets},
+				exports: []Type{TypeTargets},
+			},
+		},
+		{
+			name:     "loki.echo",
+			expected: Metadata{exports: []Type{TypeLokiLogs}},
+		},
+		{
+			name: "loki.source.file",
+			expected: Metadata{
+				accepts: []Type{TypeTargets, TypeLokiLogs},
+			},
+		},
+		{
+			name: "loki.process",
+			expected: Metadata{
+				accepts: []Type{TypeLokiLogs},
+				exports: []Type{TypeLokiLogs},
+			},
+		},
+		{
+			name: "prometheus.relabel",
+			expected: Metadata{
+				accepts: []Type{TypePromMetricsReceiver},
+				exports: []Type{TypePromMetricsReceiver},
+			},
+		},
+		{
+			name: "prometheus.remote_write",
+			expected: Metadata{
+				accepts: []Type{},
+				exports: []Type{TypePromMetricsReceiver},
+			},
+		},
+		{
+			name: "otelcol.exporter.otlp",
+			expected: Metadata{
+				accepts: []Type{},
+				exports: []Type{TypeOTELReceiver},
+			},
+		},
+		{
+			name: "otelcol.processor.filter",
+			expected: Metadata{
+				accepts: []Type{TypeOTELReceiver},
+				exports: []Type{TypeOTELReceiver},
+			},
+		},
+		{
+			name: "faro.receiver",
+			expected: Metadata{
+				accepts: []Type{TypeLokiLogs, TypeOTELReceiver},
+				exports: []Type{},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual, err := ForComponent(tt.name)
+			require.NoError(t, err)
+
+			compareSlices := func(expected, actual []Type, name string) {
+				require.Equal(t, len(expected), len(actual), "expected %d %s types, got %d; expected: %v, actual: %v", len(expected), name, len(actual), expected, actual)
+				for i := range expected {
+					require.Equal(t, expected[i].Name, actual[i].Name, "expected %s type at %d to be %q, got %q", name, i, expected[i].Name, actual[i].Name)
+				}
+			}
+
+			compareSlices(tt.expected.AllTypesAccepted(), actual.AllTypesAccepted(), "accepted")
+			compareSlices(tt.expected.AllTypesExported(), actual.AllTypesExported(), "exported")
+		})
+	}
+}

component/pyroscope/ebpf/ebpf_placeholder.go

@@ -1,4 +1,4 @@
-//go:build linux && !arm64 && !amd64
+//go:build !(linux && (arm64 || amd64))
 
 package ebpf
 
@@ -26,7 +26,7 @@ type Component struct {
 }
 
 func New(opts component.Options, args Arguments) (component.Component, error) {
-	level.Warn(opts.Logger).Log("msg", "the pyroscope.ebpf component only works on linux; enabling it otherwise will do nothing")
+	level.Warn(opts.Logger).Log("msg", "the pyroscope.ebpf component only works on ARM64 and AMD64 Linux platforms; enabling it otherwise will do nothing")
 	return &Component{}, nil
 }
 

component/registry.go

@@ -10,6 +10,8 @@ import (
 	"github.com/grafana/regexp"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel/trace"
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
 )
 
 // The parsedName of a component is the parts of its name ("remote.http") split
@@ -202,3 +204,9 @@ func Get(name string) (Registration, bool) {
 	r, ok := registered[name]
 	return r, ok
 }
+
+func AllNames() []string {
+	keys := maps.Keys(registered)
+	slices.Sort(keys)
+	return keys
+}