diff --git a/pkg/resources/statefulsets/minio-statefulset.go b/pkg/resources/statefulsets/minio-statefulset.go index 7124e351a9a..4ab8d20ae0e 100644 --- a/pkg/resources/statefulsets/minio-statefulset.go +++ b/pkg/resources/statefulsets/minio-statefulset.go @@ -786,8 +786,6 @@ func getSideCarContainer(t *miniov2.Tenant, pool *miniov2.Pool) corev1.Container "sidecar", "--tenant", t.Name, - "--config-name", - t.Spec.Configuration.Name, }, Env: []corev1.EnvVar{ { diff --git a/sidecar/cmd/sidecar/sidecar.go b/sidecar/cmd/sidecar/sidecar.go index 0f59bf2ce78..f2c1ac04ce5 100644 --- a/sidecar/cmd/sidecar/sidecar.go +++ b/sidecar/cmd/sidecar/sidecar.go @@ -36,11 +36,6 @@ var sidecarCmd = cli.Command{ Value: "", Usage: "name of tenant being validated", }, - cli.StringFlag{ - Name: "config-name", - Value: "", - Usage: "secret being watched", - }, }, } @@ -50,10 +45,5 @@ func startSideCar(ctx *cli.Context) { log.Println("Must pass --tenant flag") os.Exit(1) } - configName := ctx.String("config-name") - if configName == "" { - log.Println("Must pass --config-name flag") - os.Exit(1) - } - sidecar.StartSideCar(tenantName, configName) + sidecar.StartSideCar(tenantName) } diff --git a/pkg/configuration/tenant_configuration.go b/sidecar/pkg/configuration/tenant_configuration.go similarity index 72% rename from pkg/configuration/tenant_configuration.go rename to sidecar/pkg/configuration/tenant_configuration.go index 2900a6a19c6..c00c24aad0e 100644 --- a/pkg/configuration/tenant_configuration.go +++ b/sidecar/pkg/configuration/tenant_configuration.go @@ -17,8 +17,12 @@ package configuration import ( + "context" + "errors" "fmt" + "log" "sort" + "strconv" "strings" miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" @@ -31,27 +35,69 @@ const ( bucketDNSEnv = "MINIO_DNS_WEBHOOK_ENDPOINT" ) +type ( + secretFunc func(ctx context.Context, name string) (*corev1.Secret, error) + configFunc func(ctx context.Context, name string) (*corev1.ConfigMap, error) +) + +// TenantResources returns maps for all configmap/secret resources that +// are used in the tenant specification +func TenantResources(ctx context.Context, tenant *miniov2.Tenant, cf configFunc, sf secretFunc) (map[string]*corev1.ConfigMap, map[string]*corev1.Secret, error) { + configMaps := make(map[string]*corev1.ConfigMap) + secrets := make(map[string]*corev1.Secret) + + for _, env := range tenant.Spec.Env { + if env.ValueFrom != nil { + if env.ValueFrom.SecretKeyRef != nil { + secret, err := sf(ctx, env.ValueFrom.SecretKeyRef.Name) + if err != nil { + return nil, nil, err + } + secrets[env.ValueFrom.SecretKeyRef.Name] = secret + } + if env.ValueFrom.ConfigMapKeyRef != nil { + configmap, err := cf(ctx, env.ValueFrom.ConfigMapKeyRef.Name) + if err != nil { + return nil, nil, err + } + configMaps[env.ValueFrom.ConfigMapKeyRef.Name] = configmap + } + if env.ValueFrom.FieldRef != nil { + return nil, nil, errors.New("mapping fields is not supported") + } + if env.ValueFrom.ResourceFieldRef != nil { + return nil, nil, errors.New("mapping resource fields is not supported") + } + } + } + + secret, err := sf(ctx, tenant.Spec.Configuration.Name) + if err != nil { + return nil, nil, err + } + secrets[tenant.Spec.Configuration.Name] = secret + + return configMaps, secrets, nil +} + // GetFullTenantConfig returns the full configuration for the tenant considering the secret and the tenant spec -func GetFullTenantConfig(tenant *miniov2.Tenant, configSecret *corev1.Secret) (string, bool, bool) { +func GetFullTenantConfig(tenant *miniov2.Tenant, configMaps map[string]*corev1.ConfigMap, secrets map[string]*corev1.Secret) (string, bool, bool) { + configSecret := secrets[tenant.Spec.Configuration.Name] + seededVars := parseConfEnvSecret(configSecret) rootUserFound := false rootPwdFound := false for _, env := range seededVars { - if env.Name == "MINIO_ROOT_USER" { + if env.Name == "MINIO_ROOT_USER" || env.Name == "MINIO_ACCESS_KEY" { rootUserFound = true } - if env.Name == "MINIO_ACCESS_KEY" { - rootUserFound = true - } - if env.Name == "MINIO_ROOT_PASSWORD" { - rootPwdFound = true - } - if env.Name == "MINIO_SECRET_KEY" { + if env.Name == "MINIO_ROOT_PASSWORD" || env.Name == "MINIO_SECRET_KEY" { rootPwdFound = true } } + compiledConfig := buildTenantEnvs(tenant, seededVars) - configurationFileContent := envVarsToFileContent(compiledConfig) + configurationFileContent := envVarsToFileContent(compiledConfig, configMaps, secrets) return configurationFileContent, rootUserFound, rootPwdFound } @@ -70,12 +116,15 @@ func parseConfEnvSecret(secret *corev1.Secret) map[string]corev1.EnvVar { parts := strings.SplitN(line, "=", 2) if len(parts) == 2 { name := strings.TrimSpace(parts[0]) - value := strings.Trim(strings.TrimSpace(parts[1]), "\"") - envVar := corev1.EnvVar{ + value, err := strconv.Unquote(strings.TrimSpace(parts[1])) + if err != nil { + log.Printf("Syntax error for variable %s (skipped): %s", name, err) + continue + } + envMap[name] = corev1.EnvVar{ Name: name, Value: value, } - envMap[name] = envVar } } } @@ -234,10 +283,19 @@ func buildTenantEnvs(tenant *miniov2.Tenant, cfgEnvExisting map[string]corev1.En return envVars } -func envVarsToFileContent(envVars []corev1.EnvVar) string { - content := "" +func envVarsToFileContent(envVars []corev1.EnvVar, configMaps map[string]*corev1.ConfigMap, secrets map[string]*corev1.Secret) string { + var sb strings.Builder for _, env := range envVars { - content += fmt.Sprintf("export %s=\"%s\"\n", env.Name, env.Value) + value := env.Value + if env.ValueFrom != nil { + if env.ValueFrom.ConfigMapKeyRef != nil { + value = configMaps[env.ValueFrom.ConfigMapKeyRef.Name].Data[env.ValueFrom.ConfigMapKeyRef.Key] + } + if env.ValueFrom.SecretKeyRef != nil { + value = string(secrets[env.ValueFrom.SecretKeyRef.Name].Data[env.ValueFrom.SecretKeyRef.Key]) + } + } + sb.WriteString(fmt.Sprintf("export %s=\"%s\"\n", env.Name, value)) } - return content + return sb.String() } diff --git a/pkg/configuration/tenant_configuration_test.go b/sidecar/pkg/configuration/tenant_configuration_test.go similarity index 96% rename from pkg/configuration/tenant_configuration_test.go rename to sidecar/pkg/configuration/tenant_configuration_test.go index 9393d888744..042351730a5 100644 --- a/pkg/configuration/tenant_configuration_test.go +++ b/sidecar/pkg/configuration/tenant_configuration_test.go @@ -67,7 +67,7 @@ export MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRy } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := envVarsToFileContent(tt.args.envVars); got != tt.want { + if got := envVarsToFileContent(tt.args.envVars, nil, nil); got != tt.want { t.Errorf("envVarsToFileContent() = `%v`, want `%v`", got, tt.want) } }) @@ -377,7 +377,12 @@ export TEST="value" for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.args.tenant.EnsureDefaults() - if got, _, _ := GetFullTenantConfig(tt.args.tenant, tt.args.configSecret); got != tt.want { + + var configMaps map[string]*corev1.ConfigMap + secrets := map[string]*corev1.Secret{ + tt.args.tenant.ConfigurationSecretName(): tt.args.configSecret, + } + if got, _, _ := GetFullTenantConfig(tt.args.tenant, configMaps, secrets); got != tt.want { t.Errorf("GetFullTenantConfig() = `%v`, want `%v`", got, tt.want) } }) diff --git a/sidecar/pkg/sidecar/sidecar_utils.go b/sidecar/pkg/sidecar/sidecar_utils.go index dcbaa86377e..bf37cccc88a 100644 --- a/sidecar/pkg/sidecar/sidecar_utils.go +++ b/sidecar/pkg/sidecar/sidecar_utils.go @@ -22,9 +22,10 @@ import ( "log" "net/http" "os" + "sync" "time" - "github.com/minio/operator/pkg/configuration" + "github.com/minio/operator/sidecar/pkg/configuration" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,7 +47,7 @@ func init() { } // StartSideCar instantiates kube clients and starts the side-car controller -func StartSideCar(tenantName string, secretName string) { +func StartSideCar(tenantName string) { log.Println("Starting Sidecar") var cfg *rest.Config var err error @@ -90,7 +91,7 @@ func StartSideCar(tenantName string, secretName string) { tenant.EnsureDefaults() - controller := NewSideCarController(kubeClient, controllerClient, namespace, tenantName, secretName) + controller := NewSideCarController(kubeClient, controllerClient, tenant) controller.ws = configureWebhookServer(controller) controller.probeServer = configureProbesServer(tenant) controller.sidecar = configureSidecarServer(controller) @@ -134,36 +135,38 @@ func StartSideCar(tenantName string, secretName string) { type Controller struct { kubeClient *kubernetes.Clientset controllerClient *clientset.Clientset - tenantName string - secretName string minInformerFactory minioInformers.SharedInformerFactory secretInformer coreinformers.SecretInformer + configMapInformer coreinformers.ConfigMapInformer tenantInformer v22.TenantInformer - namespace string informerFactory informers.SharedInformerFactory + lock sync.Mutex + tenant *v2.Tenant + configMaps map[string]*corev1.ConfigMap + secrets map[string]*corev1.Secret ws *http.Server probeServer *http.Server sidecar *http.Server } // NewSideCarController returns an instance of Controller with the provided clients -func NewSideCarController(kubeClient *kubernetes.Clientset, controllerClient *clientset.Clientset, namespace string, tenantName string, secretName string) *Controller { - factory := informers.NewSharedInformerFactoryWithOptions(kubeClient, time.Hour*1, informers.WithNamespace(namespace)) +func NewSideCarController(kubeClient *kubernetes.Clientset, controllerClient *clientset.Clientset, tenant *v2.Tenant) *Controller { + factory := informers.NewSharedInformerFactoryWithOptions(kubeClient, time.Hour*1, informers.WithNamespace(tenant.Namespace)) secretInformer := factory.Core().V1().Secrets() + configMapInformer := factory.Core().V1().ConfigMaps() - minioInformerFactory := minioInformers.NewSharedInformerFactoryWithOptions(controllerClient, time.Hour*1, minioInformers.WithNamespace(namespace)) + minioInformerFactory := minioInformers.NewSharedInformerFactoryWithOptions(controllerClient, time.Hour*1, minioInformers.WithNamespace(tenant.Namespace)) tenantInformer := minioInformerFactory.Minio().V2().Tenants() c := &Controller{ kubeClient: kubeClient, controllerClient: controllerClient, - tenantName: tenantName, - namespace: namespace, - secretName: secretName, minInformerFactory: minioInformerFactory, informerFactory: factory, tenantInformer: tenantInformer, secretInformer: secretInformer, + configMapInformer: configMapInformer, + tenant: tenant, } _, err := tenantInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -175,7 +178,13 @@ func NewSideCarController(kubeClient *kubernetes.Clientset, controllerClient *cl // Two different versions of the same Tenant will always have different RVs. return } - c.regenCfgWithTenant(newTenant) + c.lock.Lock() + defer c.lock.Unlock() + + log.Println("tenant was updated, regenerating configuration") + + c.tenant = newTenant + c.regenCfg() }, }) if err != nil { @@ -186,19 +195,22 @@ func NewSideCarController(kubeClient *kubernetes.Clientset, controllerClient *cl _, err = secretInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ UpdateFunc: func(old, new interface{}) { oldSecret := old.(*corev1.Secret) - // ignore anything that is not what we want - if oldSecret.Name != secretName { - return - } - log.Printf("Config secret '%s' sync", secretName) newSecret := new.(*corev1.Secret) - if newSecret.ResourceVersion == oldSecret.ResourceVersion { - // Periodic resync will send update events for all known Tenants. - // Two different versions of the same Tenant will always have different RVs. + if oldSecret.ResourceVersion == newSecret.ResourceVersion { + // Periodic resync will send update events for all known secrets. + // Two different versions of the same secret will always have different RVs. return } + c.lock.Lock() + defer c.lock.Unlock() - c.regenCfgWithSecret(newSecret) + log.Printf("secret %s was updated, regenerating configuration", newSecret.Name) + + if _, ok := c.secrets[oldSecret.Name]; !ok { + // Not interested in secrets that we don't use + return + } + c.regenCfg() }, }) if err != nil { @@ -206,43 +218,60 @@ func NewSideCarController(kubeClient *kubernetes.Clientset, controllerClient *cl return nil } - return c -} + _, err = configMapInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(old, new interface{}) { + oldConfigMap := old.(*corev1.ConfigMap) + newConfigMap := new.(*corev1.ConfigMap) + if oldConfigMap.ResourceVersion == newConfigMap.ResourceVersion { + // Periodic resync will send update events for all known config maps. + // Two different versions of the same config map will always have different RVs. + return + } + c.lock.Lock() + defer c.lock.Unlock() -func (c *Controller) regenCfgWithTenant(tenant *v2.Tenant) { - // get the tenant secret - tenant.EnsureDefaults() + log.Printf("configmap %s was updated, regenerating configuration", newConfigMap.Name) - configSecret, err := c.secretInformer.Lister().Secrets(c.namespace).Get(tenant.Spec.Configuration.Name) + if _, ok := c.configMaps[oldConfigMap.Name]; !ok { + // Not interested in configmaps that we don't use + return + } + c.regenCfg() + }, + }) if err != nil { - log.Println("could not get secret", err) - return + log.Println("could not add event handler for secret informer", err) + return nil } - fileContents, rootUserFound, rootPwdFound := configuration.GetFullTenantConfig(tenant, configSecret) + return c +} - if !rootUserFound || !rootPwdFound { - log.Println("Missing root credentials in the configuration.") - log.Println("MinIO won't start") - os.Exit(1) - } +func (c *Controller) getSecret(_ context.Context, name string) (*corev1.Secret, error) { + return c.secretInformer.Lister().Secrets(c.tenant.Namespace).Get(name) +} - err = os.WriteFile(v2.CfgFile, []byte(fileContents), 0o644) - if err != nil { - log.Println(err) - } +func (c *Controller) getConfigMap(_ context.Context, name string) (*corev1.ConfigMap, error) { + return c.configMapInformer.Lister().ConfigMaps(c.tenant.Namespace).Get(name) } -func (c *Controller) regenCfgWithSecret(configSecret *corev1.Secret) { - // get the tenant - tenant, err := c.tenantInformer.Lister().Tenants(c.namespace).Get(c.tenantName) +func (c *Controller) regenCfg() { + // get the tenant secret + c.tenant.EnsureDefaults() + + // determine the configmaps and secrets to watch + configMaps, secrets, err := configuration.TenantResources(context.Background(), c.tenant, c.getConfigMap, c.getSecret) if err != nil { - log.Println("could not get secret", err) + log.Println(err) return } - tenant.EnsureDefaults() - fileContents, rootUserFound, rootPwdFound := configuration.GetFullTenantConfig(tenant, configSecret) + // update secrets and configmaps that should be watched + c.secrets = secrets + c.configMaps = configMaps + + // obtain the full tenant configuration + fileContents, rootUserFound, rootPwdFound := configuration.GetFullTenantConfig(c.tenant, c.configMaps, c.secrets) if !rootUserFound || !rootPwdFound { log.Println("Missing root credentials in the configuration.") @@ -250,7 +279,14 @@ func (c *Controller) regenCfgWithSecret(configSecret *corev1.Secret) { os.Exit(1) } - err = os.WriteFile(v2.CfgFile, []byte(fileContents), 0o644) + tmpFile := v2.CfgFile + ".tmp" + defer os.Remove(tmpFile) + + err = os.WriteFile(tmpFile, []byte(fileContents), 0o644) + if err != nil { + log.Println(err) + } + err = os.Rename(tmpFile, v2.CfgFile) if err != nil { log.Println(err) } @@ -263,8 +299,13 @@ func (c *Controller) Run(stopCh chan struct{}) error { go c.informerFactory.Start(stopCh) // wait for the initial synchronization of the local cache. - if !cache.WaitForCacheSync(stopCh, c.tenantInformer.Informer().HasSynced, c.secretInformer.Informer().HasSynced) { + if !cache.WaitForCacheSync(stopCh, c.tenantInformer.Informer().HasSynced, c.configMapInformer.Informer().HasSynced, c.secretInformer.Informer().HasSynced) { return fmt.Errorf("failed to sync") } + + c.lock.Lock() + defer c.lock.Unlock() + + c.regenCfg() return nil } diff --git a/sidecar/pkg/validator/validator.go b/sidecar/pkg/validator/validator.go index 94671a29f33..df8c7670c58 100644 --- a/sidecar/pkg/validator/validator.go +++ b/sidecar/pkg/validator/validator.go @@ -23,11 +23,12 @@ import ( "os" "strings" - "github.com/minio/operator/pkg/configuration" + "github.com/minio/operator/sidecar/pkg/configuration" "k8s.io/client-go/kubernetes" miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" operatorClientset "github.com/minio/operator/pkg/client/clientset/versioned" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "k8s.io/klog/v2" @@ -66,14 +67,19 @@ func Validate(tenantName string) { panic(err) } tenant.EnsureDefaults() - // get tenant config secret - configSecret, err := kubeClient.CoreV1().Secrets(namespace).Get(ctx, tenant.Spec.Configuration.Name, metav1.GetOptions{}) + + // determine the configmaps and secrets to watch + configMaps, secrets, err := configuration.TenantResources(context.Background(), tenant, func(ctx context.Context, name string) (*corev1.ConfigMap, error) { + return kubeClient.CoreV1().ConfigMaps(tenant.Namespace).Get(ctx, name, metav1.GetOptions{}) + }, func(ctx context.Context, name string) (*corev1.Secret, error) { + return kubeClient.CoreV1().Secrets(tenant.Namespace).Get(ctx, name, metav1.GetOptions{}) + }) if err != nil { log.Println(err) panic(err) } - fileContents, rootUserFound, rootPwdFound := configuration.GetFullTenantConfig(tenant, configSecret) + fileContents, rootUserFound, rootPwdFound := configuration.GetFullTenantConfig(tenant, configMaps, secrets) if !rootUserFound || !rootPwdFound { log.Println("Missing root credentials in the configuration.")