Merge branch 'master' into graylog-cluster-centralized-view

README.markdown

@@ -2,7 +2,6 @@
 
 [![License](https://img.shields.io/badge/license-SSPL-green)](https://www.mongodb.com/licensing/server-side-public-license)
 [![Maven Central](https://img.shields.io/maven-central/v/org.graylog2/graylog2-server.svg)](http://mvnrepository.com/artifact/org.graylog2/graylog2-server)
-[![Build](https://github.com/Graylog2/graylog2-server/actions/workflows/build.yml/badge.svg)](https://github.com/Graylog2/graylog2-server/actions/workflows/build.yml)
 
 Welcome! _Graylog_ is a free and open log management platform.
 

UPGRADING.md

@@ -23,9 +23,9 @@ Alternatively the `SaveOrCancelButtons` component can be used in the edit compon
 
 The following Java Code API changes have been made.
 
-| File/method                         | Description            |
-|-------------------------------------|------------------------|
-| `tbd`                               | tbd                    |
+| File/method                                    | Description |
+|------------------------------------------------|-------------|
+| `org.graylog.scheduler.JobSchedule#toDBUpdate` | removed     |
 
 ## REST API Endpoint Changes
 

changelog/unreleased/issue-18563.toml

@@ -0,0 +1,5 @@
+type = "fixed"
+message = "Improve performance of close and delete actions when working with a lot of index sets."
+
+issues = ["18563"]
+pulls = ["21195"]

changelog/unreleased/issue-19058.toml

@@ -0,0 +1,5 @@
+type = "fixed"
+message = "Fixing highlighting of message in message table by id."
+
+issues = ["19058"]
+pulls = ["21389"]

changelog/unreleased/issue-19287.toml

@@ -0,0 +1,5 @@
+type = "c"
+message = "Log usage of deprecated pipeline functions, specifically `remove_field`."
+
+issues = ["19287"]
+pulls = ["21386"]

changelog/unreleased/issue-19975.toml

@@ -0,0 +1,5 @@
+type = "f"
+message = "Catch and report exceptions during grok pattern matching."
+
+issues=["19975"]
+pulls = ["21290"]

changelog/unreleased/issue-20955.toml

@@ -0,0 +1,5 @@
+type = "fixed"
+message = "Fix unescaped double quotes in map and collection typed fields in Custom HTTP Notification JSON body."
+
+issues = ["20955"]
+pulls = ["21167"]

changelog/unreleased/issue-20965.toml

@@ -0,0 +1,5 @@
+type = "a"
+message = "Add unit handling to a single number widget"
+
+issues = ["20965"]
+pulls = ["21339"]

changelog/unreleased/issue-21015.toml

@@ -0,0 +1,5 @@
+type = "f"
+message = "Handle path prefix when serving web interface assets."
+
+issues = ["21015"]
+pulls = ["21104"]

changelog/unreleased/issue-21145.toml

@@ -0,0 +1,5 @@
+type = "f"
+message = "Fix when widget title input in widget edit mode looses cursor position on change"
+
+pulls = ["21327"]
+issues = ["21145"]

changelog/unreleased/issue-21185.toml

@@ -0,0 +1,5 @@
+type = "f"
+message = "Fix displaying very small percentages."
+
+issues = ["21185"]
+pulls = ["21368"]

changelog/unreleased/issue-21223.toml

@@ -0,0 +1,5 @@
+type = "fixed"
+message = "Fix JNA temp directory to be located in the configured server data dir."
+
+issues = ["21223"]
+pulls = ["21298"]

changelog/unreleased/issue-21248.toml

@@ -0,0 +1,4 @@
+type = "f"
+message = "Add error handling for event definition bulk action on general perspective."
+
+pulls = ["21248"]

changelog/unreleased/issue-21351.toml

@@ -0,0 +1,5 @@
+type = "f"
+message = "Add default auto refresh config on events and alerts page"
+
+issues = ["21350"]
+pulls = ["21351"]

changelog/unreleased/issue-21398.toml

@@ -0,0 +1,5 @@
+type = "f"
+message = "Fix an issue causing saving searches/dashboards after clearing the unit type to fail with error."
+
+pulls = ["21399"]
+issues = ["21398"]

changelog/unreleased/issue-9368.toml

@@ -0,0 +1,5 @@
+type = "fixed"
+message = "Fixed query validation with parameter usage. Parameters representing field names do not cause 'unknown field' warnings anymore."
+
+issues = ["9368"]
+pulls = ["21469"]

changelog/unreleased/pr-21056.toml

@@ -0,0 +1,5 @@
+type = "a"
+message = "Allowing to filter events based on event definition, priority, key & aggregation time range."
+
+issues = ["21055"]
+pulls = ["21056"]

changelog/unreleased/pr-21062.toml

@@ -0,0 +1,4 @@
+type = "c"
+message = "Better handling of intermediate CAs in datanode truststore"
+
+pulls = ["21062"]

changelog/unreleased/pr-21123.toml

@@ -0,0 +1,4 @@
+type = "c"
+message = "Adjust Event Priority field choices for consistency with Graylog Security."
+
+pulls = ["21123"]

changelog/unreleased/pr-21197.toml

@@ -0,0 +1,5 @@
+type = "fixed"
+message = "Add archive restore retry on mapper parsing exception."
+
+issues = ["graylog-plugin-enterprise#9208"]
+pulls = ["21197", "Graylog2/graylog-plugin-enterprise#9413"]

changelog/unreleased/pr-21205.toml

@@ -0,0 +1,8 @@
+type = "a"
+message = "Enable defining required permissions for navigation web interface plugin."
+
+pulls = ["21205"]
+details.user = """
+Before it was only possible to define required permissions for a navigation dropdown item.
+"""
+

changelog/unreleased/pr-21206.toml

@@ -0,0 +1,5 @@
+type = "f"
+message = "Filters system event definitions from list of entities that can be exported in a content pack."
+
+pulls = ["21206"]
+issues = ["21166"]

changelog/unreleased/pr-21208.toml

@@ -0,0 +1,4 @@
+type = "f"
+message = "batching request for index block status if the combined length of the indices exceed the max possible URL length "
+
+pulls = ["21208"]

changelog/unreleased/pr-21217.toml

@@ -0,0 +1,4 @@
+type = "f"
+message = "Fixed bug where inputs with multiple encrypted configuration values could not be saved."
+
+pulls = ["21217"]

changelog/unreleased/pr-21238.toml

@@ -0,0 +1,4 @@
+type = "a"
+message = "Enabling bulk actions for event definitions."
+
+pulls = ["21238"]

changelog/unreleased/pr-21262.toml

@@ -0,0 +1,4 @@
+type = "a"
+message = "Adding 'Replay Search' Bulk action to alerts & events."
+
+pulls = ["21262"]

changelog/unreleased/pr-21286.toml

@@ -0,0 +1,4 @@
+type = "fixed"
+message = "Fix `org.graylog2.outputs.ElasticSearchOutput.writes` metric."
+
+pulls = ["21286"]

changelog/unreleased/pr-21367.toml

@@ -0,0 +1,5 @@
+type = "fixed"
+message = "Adding missing API descriptions on method parameters for Simple Scripting API."
+
+pulls = ["21367"]
+issues = ["20821"]

changelog/unreleased/pr-21454.toml

@@ -0,0 +1,4 @@
+type = "a"
+message = "Added metrics to the shared \"scheduler\" and \"daemonScheduler\" thread pools."
+
+pulls = ["21454"]

data-node/migration/env

@@ -8,7 +8,7 @@ GRAYLOG_PASSWORD_SECRET=
 # system and in case you lose connectivity to your authentication backend)
 # This password cannot be changed using the API or via the web interface. If you need to change it,
 # modify it in this file.
-# Create one by using for example: echo -n yourpassword | shasum -a 256
+# Create one by using for example: echo -n yourpassword | sha256sum
 # and put the resulting hash value into the following line
 # CHANGE THIS!
 GRAYLOG_ROOT_PASSWORD_SHA2=

data-node/pom.xml

@@ -835,6 +835,25 @@
                 <groupId>com.mycila</groupId>
                 <artifactId>license-maven-plugin</artifactId>
             </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-enforcer-plugin</artifactId>
+                <configuration>
+                    <rules>
+                        <RestrictImports>
+                            <groups>
+                                <group>
+                                    <reason>Data Node packages must be allowed for the data-node/ module.</reason>
+                                    <allowedImports>
+                                        <allowedImport>org.graylog.datanode.**</allowedImport>
+                                    </allowedImports>
+                                </group>
+                            </groups>
+                        </RestrictImports>
+                    </rules>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 

data-node/src/main/java/org/graylog/datanode/Configuration.java

@@ -32,6 +32,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.net.InetAddresses;
 import org.graylog.datanode.configuration.DatanodeDirectories;
+import org.graylog2.CommonNodeConfiguration;
 import org.graylog2.Configuration.SafeClassesValidator;
 import org.graylog2.configuration.Documentation;
 import org.graylog2.plugin.Tools;
@@ -57,7 +58,7 @@
  * Helper class to hold configuration of DataNode
  */
 @SuppressWarnings("FieldMayBeFinal")
-public class Configuration {
+public class Configuration implements CommonNodeConfiguration {
     private static final Logger LOG = LoggerFactory.getLogger(Configuration.class);
     public static final String TRANSPORT_CERTIFICATE_PASSWORD_PROPERTY = "transport_certificate_password";
     public static final String HTTP_CERTIFICATE_PASSWORD_PROPERTY = "http_certificate_password";
@@ -659,4 +660,14 @@ public List<String> getNodeRoles() {
     public String getOpensearchHeap() {
         return opensearchHeap;
     }
+
+    @Override
+    public String getEnvironmentVariablePrefix() {
+        return "GRAYLOG_DATANODE_";
+    }
+
+    @Override
+    public String getSystemPropertyPrefix() {
+        return "graylog.datanode.";
+    }
 }

data-node/src/main/java/org/graylog/datanode/bindings/ServerBindings.java → data-node/src/main/java/org/graylog/datanode/bindings/DatanodeServerBindings.java

@@ -16,61 +16,33 @@
  */
 package org.graylog.datanode.bindings;
 
-import com.google.common.eventbus.EventBus;
 import com.google.inject.TypeLiteral;
-import com.google.inject.multibindings.Multibinder;
 import com.google.inject.multibindings.OptionalBinder;
-import jakarta.ws.rs.container.DynamicFeature;
-import jakarta.ws.rs.ext.ExceptionMapper;
-import org.graylog.datanode.Configuration;
 import org.graylog.datanode.shared.system.activities.DataNodeActivityWriter;
-import org.graylog2.bindings.providers.ClusterEventBusProvider;
 import org.graylog2.cluster.ClusterConfigServiceImpl;
 import org.graylog2.cluster.nodes.DataNodeClusterService;
 import org.graylog2.cluster.nodes.DataNodeDto;
 import org.graylog2.cluster.nodes.NodeService;
-import org.graylog2.events.ClusterEventBus;
-import org.graylog2.jackson.InputConfigurationBeanDeserializerModifier;
 import org.graylog2.plugin.cluster.ClusterConfigService;
 import org.graylog2.plugin.cluster.ClusterIdFactory;
 import org.graylog2.plugin.cluster.RandomUUIDClusterIdFactory;
 import org.graylog2.plugin.inject.Graylog2Module;
-import org.graylog2.shared.bindings.providers.EventBusProvider;
 import org.graylog2.shared.system.activities.ActivityWriter;
 
-public class ServerBindings extends Graylog2Module {
-    private final Configuration configuration;
-    private final boolean isMigrationCommand;
+public class DatanodeServerBindings extends Graylog2Module {
 
-    public ServerBindings(Configuration configuration, boolean isMigrationCommand) {
-
-        this.configuration = configuration;
-        this.isMigrationCommand = isMigrationCommand;
+    public DatanodeServerBindings() {
     }
 
     @Override
     protected void configure() {
         bindInterfaces();
         bindSingletons();
 
-        bindProviders();
-        bindFactoryModules();
         bindDynamicFeatures();
         bindExceptionMappers();
-        bindAdditionalJerseyComponents();
-//        install(new AuthenticatingRealmModule(configuration));
-//        install(new AuthorizationOnlyRealmModule());
     }
 
-    private void bindProviders() {
-        bind(ClusterEventBus.class).toProvider(ClusterEventBusProvider.class).asEagerSingleton();
-        bind(EventBus.class).toProvider(EventBusProvider.class).asEagerSingleton();
-        bind(InputConfigurationBeanDeserializerModifier.class).toInstance(InputConfigurationBeanDeserializerModifier.withoutConfig());
-    }
-
-    private void bindFactoryModules() {
-        // System Jobs
-    }
 
     private void bindSingletons() {
         bind(ClusterConfigService.class).to(ClusterConfigServiceImpl.class).asEagerSingleton();
@@ -83,14 +55,11 @@ private void bindInterfaces() {
     }
 
     private void bindDynamicFeatures() {
-        final Multibinder<Class<? extends DynamicFeature>> dynamicFeatures = jerseyDynamicFeatureBinder();
+        jerseyDynamicFeatureBinder();
     }
 
     private void bindExceptionMappers() {
-        final Multibinder<Class<? extends ExceptionMapper>> exceptionMappers = jerseyExceptionMapperBinder();
+        jerseyExceptionMapperBinder();
     }
 
-    private void bindAdditionalJerseyComponents() {
-//        jerseyAdditionalComponentsBinder().addBinding().toInstance(GenericErrorCsvWriter.class);
-    }
 }

data-node/src/main/java/org/graylog/datanode/bindings/OpensearchProcessBindings.java

@@ -18,23 +18,33 @@
 
 import com.google.common.util.concurrent.Service;
 import com.google.inject.AbstractModule;
+import com.google.inject.TypeLiteral;
 import com.google.inject.multibindings.Multibinder;
 import org.graylog.datanode.configuration.DatanodeTrustManagerProvider;
 import org.graylog.datanode.configuration.OpensearchConfigurationService;
+import org.graylog.datanode.configuration.variants.DatanodeKeystoreOpensearchCertificatesProvider;
+import org.graylog.datanode.configuration.variants.LocalConfigurationCertificatesProvider;
+import org.graylog.datanode.configuration.variants.NoOpensearchCertificatesProvider;
+import org.graylog.datanode.configuration.variants.OpensearchCertificatesProvider;
 import org.graylog.datanode.metrics.ConfigureMetricsIndexSettings;
 import org.graylog.datanode.opensearch.OpensearchProcess;
 import org.graylog.datanode.opensearch.OpensearchProcessImpl;
 import org.graylog.datanode.opensearch.OpensearchProcessService;
+import org.graylog.datanode.opensearch.configuration.OpensearchConfigurationParams;
 import org.graylog.datanode.opensearch.configuration.OpensearchUsableSpace;
 import org.graylog.datanode.opensearch.configuration.OpensearchUsableSpaceProvider;
-import org.graylog.datanode.opensearch.configuration.beans.OpensearchConfigurationBean;
+import org.graylog.datanode.opensearch.configuration.beans.impl.OpensearchClusterConfigurationBean;
+import org.graylog.datanode.opensearch.configuration.beans.impl.OpensearchCommonConfigurationBean;
+import org.graylog.datanode.opensearch.configuration.beans.impl.OpensearchDefaultConfigFilesBean;
+import org.graylog.datanode.opensearch.configuration.beans.impl.OpensearchSecurityConfigurationBean;
 import org.graylog.datanode.opensearch.configuration.beans.impl.SearchableSnapshotsConfigurationBean;
 import org.graylog.datanode.opensearch.statemachine.OpensearchStateMachine;
 import org.graylog.datanode.opensearch.statemachine.OpensearchStateMachineProvider;
 import org.graylog.datanode.opensearch.statemachine.tracer.ClusterNodeStateTracer;
 import org.graylog.datanode.opensearch.statemachine.tracer.OpensearchWatchdog;
 import org.graylog.datanode.opensearch.statemachine.tracer.StateMachineTracer;
 import org.graylog.datanode.opensearch.statemachine.tracer.StateMachineTransitionLogger;
+import org.graylog.datanode.process.configuration.beans.DatanodeConfigurationBean;
 
 public class OpensearchProcessBindings extends AbstractModule {
 
@@ -49,9 +59,21 @@ protected void configure() {
 
         bind(OpensearchUsableSpace.class).toProvider(OpensearchUsableSpaceProvider.class).asEagerSingleton();
 
-        //opensearch configuration beans
-        Multibinder<OpensearchConfigurationBean> opensearchConfigurationBeanMultibinder = Multibinder.newSetBinder(binder(), OpensearchConfigurationBean.class);
+        //opensearch certificate providers
+        Multibinder<OpensearchCertificatesProvider> opensearchCertificatesProviders = Multibinder.newSetBinder(binder(), OpensearchCertificatesProvider.class);
+        opensearchCertificatesProviders.addBinding().to(LocalConfigurationCertificatesProvider.class).asEagerSingleton();
+        opensearchCertificatesProviders.addBinding().to(DatanodeKeystoreOpensearchCertificatesProvider.class).asEagerSingleton();
+        opensearchCertificatesProviders.addBinding().to(NoOpensearchCertificatesProvider.class).asEagerSingleton();
+
+
+        //opensearch configuration beans. The order of the beans is important here!
+
+        Multibinder<DatanodeConfigurationBean<OpensearchConfigurationParams>> opensearchConfigurationBeanMultibinder = Multibinder.newSetBinder(binder(), new TypeLiteral<DatanodeConfigurationBean<OpensearchConfigurationParams>>() {});
+        opensearchConfigurationBeanMultibinder.addBinding().to(OpensearchDefaultConfigFilesBean.class).asEagerSingleton();
+        opensearchConfigurationBeanMultibinder.addBinding().to(OpensearchCommonConfigurationBean.class).asEagerSingleton();
+        opensearchConfigurationBeanMultibinder.addBinding().to(OpensearchClusterConfigurationBean.class).asEagerSingleton();
         opensearchConfigurationBeanMultibinder.addBinding().to(SearchableSnapshotsConfigurationBean.class).asEagerSingleton();
+        opensearchConfigurationBeanMultibinder.addBinding().to(OpensearchSecurityConfigurationBean.class).asEagerSingleton();
 
         // this service both starts and provides the opensearch process
         serviceBinder.addBinding().to(OpensearchConfigurationService.class).asEagerSingleton();