Merge remote-tracking branch 'origin/master' into refactor/remove-pro…

…p-types
Graylog2 · Jan 30, 2025 · ff92115 · ff92115
2 parents 6147464 + 60ef68f
commit ff92115
Show file tree

Hide file tree

Showing 75 changed files with 843 additions and 602 deletions.
diff --git a/changelog/unreleased/issue-21351.toml b/changelog/unreleased/issue-21351.toml
@@ -0,0 +1,5 @@
+type = "f"
+message = "Add default auto refresh config on events and alerts page"
+
+issues = ["21350"]
+pulls = ["21351"]
diff --git a/changelog/unreleased/issue-9368.toml b/changelog/unreleased/issue-9368.toml
@@ -0,0 +1,5 @@
+type = "fixed"
+message = "Fixed query validation with parameter usage. Parameters representing field names do not cause 'unknown field' warnings anymore."
+
+issues = ["9368"]
+pulls = ["21469"]
diff --git a/changelog/unreleased/pr-21454.toml b/changelog/unreleased/pr-21454.toml
@@ -0,0 +1,4 @@
+type = "a"
+message = "Added metrics to the shared \"scheduler\" and \"daemonScheduler\" thread pools."
+
+pulls = ["21454"]
diff --git a/data-node/pom.xml b/data-node/pom.xml
@@ -835,6 +835,25 @@
                 <groupId>com.mycila</groupId>
                 <artifactId>license-maven-plugin</artifactId>
             </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-enforcer-plugin</artifactId>
+                <configuration>
+                    <rules>
+                        <RestrictImports>
+                            <groups>
+                                <group>
+                                    <reason>Data Node packages must be allowed for the data-node/ module.</reason>
+                                    <allowedImports>
+                                        <allowedImport>org.graylog.datanode.**</allowedImport>
+                                    </allowedImports>
+                                </group>
+                            </groups>
+                        </RestrictImports>
+                    </rules>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 

diff --git a/...raylog/datanode/ClientCertResourceIT.java → ...ackend/datanode/ClientCertResourceIT.java b/...raylog/datanode/ClientCertResourceIT.java → ...ackend/datanode/ClientCertResourceIT.java
@@ -14,7 +14,7 @@
  * along with this program. If not, see
  * <http://www.mongodb.com/licensing/server-side-public-license>.
  */
-package org.graylog.datanode;
+package org.graylog.searchbackend.datanode;
 
 import com.jayway.jsonpath.DocumentContext;
 import com.jayway.jsonpath.JsonPath;

diff --git a/...deOpensearchProxyDisabledAllowlistIT.java → ...deOpensearchProxyDisabledAllowlistIT.java b/...deOpensearchProxyDisabledAllowlistIT.java → ...deOpensearchProxyDisabledAllowlistIT.java
@@ -14,9 +14,8 @@
  * along with this program. If not, see
  * <http://www.mongodb.com/licensing/server-side-public-license>.
  */
-package org.graylog.datanode;
+package org.graylog.searchbackend.datanode;
 
-import io.restassured.response.ValidatableResponse;
 import org.graylog.testing.completebackend.Lifecycle;
 import org.graylog.testing.completebackend.apis.GraylogApis;
 import org.graylog.testing.containermatrix.SearchServer;

diff --git a/...g/datanode/DatanodeOpensearchProxyIT.java → ...d/datanode/DatanodeOpensearchProxyIT.java b/...g/datanode/DatanodeOpensearchProxyIT.java → ...d/datanode/DatanodeOpensearchProxyIT.java
@@ -14,7 +14,7 @@
  * along with this program. If not, see
  * <http://www.mongodb.com/licensing/server-side-public-license>.
  */
-package org.graylog.datanode;
+package org.graylog.searchbackend.datanode;
 
 import com.github.rholder.retry.RetryException;
 import io.restassured.response.ValidatableResponse;

diff --git a/...ylog/datanode/DatanodeProvisioningIT.java → ...kend/datanode/DatanodeProvisioningIT.java b/...ylog/datanode/DatanodeProvisioningIT.java → ...kend/datanode/DatanodeProvisioningIT.java
@@ -14,7 +14,7 @@
  * along with this program. If not, see
  * <http://www.mongodb.com/licensing/server-side-public-license>.
  */
-package org.graylog.datanode;
+package org.graylog.searchbackend.datanode;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.github.joschi.jadconfig.util.Duration;

diff --git a/...graylog/datanode/DatanodeRestProxyIT.java → ...backend/datanode/DatanodeRestProxyIT.java b/...graylog/datanode/DatanodeRestProxyIT.java → ...backend/datanode/DatanodeRestProxyIT.java
@@ -14,7 +14,7 @@
  * along with this program. If not, see
  * <http://www.mongodb.com/licensing/server-side-public-license>.
  */
-package org.graylog.datanode;
+package org.graylog.searchbackend.datanode;
 
 import io.restassured.response.ValidatableResponse;
 import org.assertj.core.api.Assertions;

diff --git a/...datanode/DatanodeSelfsignedStartupIT.java → ...datanode/DatanodeSelfsignedStartupIT.java b/...datanode/DatanodeSelfsignedStartupIT.java → ...datanode/DatanodeSelfsignedStartupIT.java
@@ -14,7 +14,7 @@
  * along with this program. If not, see
  * <http://www.mongodb.com/licensing/server-side-public-license>.
  */
-package org.graylog.datanode;
+package org.graylog.searchbackend.datanode;
 
 import com.github.joschi.jadconfig.util.Duration;
 import com.github.rholder.retry.RetryException;
@@ -33,10 +33,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.cert.CertificateException;
 import java.util.concurrent.ExecutionException;
 
 @ContainerMatrixTestsConfiguration(serverLifecycle = Lifecycle.CLASS, searchVersions = SearchServer.DATANODE_DEV,

diff --git a/...datanode/RemoteReindexingMigrationIT.java → ...datanode/RemoteReindexingMigrationIT.java b/...datanode/RemoteReindexingMigrationIT.java → ...datanode/RemoteReindexingMigrationIT.java
@@ -14,7 +14,7 @@
  * along with this program. If not, see
  * <http://www.mongodb.com/licensing/server-side-public-license>.
  */
-package org.graylog.datanode;
+package org.graylog.searchbackend.datanode;
 
 import com.github.rholder.retry.Attempt;
 import com.github.rholder.retry.RetryException;

diff --git a/...g/elasticsearch/e2e/ElasticsearchE2E.java → ...d/elasticsearch/e2e/ElasticsearchE2E.java b/...g/elasticsearch/e2e/ElasticsearchE2E.java → ...d/elasticsearch/e2e/ElasticsearchE2E.java
@@ -14,7 +14,7 @@
  * along with this program. If not, see
  * <http://www.mongodb.com/licensing/server-side-public-license>.
  */
-package org.graylog.elasticsearch.e2e;
+package org.graylog.searchbackend.elasticsearch.e2e;
 
 import org.graylog.testing.completebackend.apis.GraylogApis;
 import org.graylog.testing.containermatrix.annotations.ContainerMatrixTest;

diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/favorites/FavoritesResource.java b/graylog2-server/src/main/java/org/graylog/plugins/views/favorites/FavoritesResource.java
@@ -19,18 +19,9 @@
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import io.swagger.annotations.ApiParam;
-import org.apache.shiro.authz.annotation.RequiresAuthentication;
-import org.graylog.grn.GRN;
-import org.graylog.plugins.views.audit.ViewsAuditEventTypes;
-import org.graylog.plugins.views.search.permissions.SearchUser;
-import org.graylog2.audit.jersey.AuditEvent;
-import org.graylog2.rest.models.PaginatedResponse;
-
 import jakarta.inject.Inject;
-
+import jakarta.validation.constraints.Min;
 import jakarta.validation.constraints.NotEmpty;
-import jakarta.validation.constraints.NotNull;
-
 import jakarta.ws.rs.DELETE;
 import jakarta.ws.rs.DefaultValue;
 import jakarta.ws.rs.GET;
@@ -41,6 +32,11 @@
 import jakarta.ws.rs.QueryParam;
 import jakarta.ws.rs.core.Context;
 import jakarta.ws.rs.core.MediaType;
+import org.apache.shiro.authz.annotation.RequiresAuthentication;
+import org.graylog.plugins.views.audit.ViewsAuditEventTypes;
+import org.graylog.plugins.views.search.permissions.SearchUser;
+import org.graylog2.audit.jersey.AuditEvent;
+import org.graylog2.rest.models.PaginatedResponse;
 
 import java.util.Optional;
 
@@ -60,8 +56,8 @@ public FavoritesResource(final FavoritesService favoritesService) {
 
     @GET
     @ApiOperation("Get the Favorites for the Start Page for the user")
-    public PaginatedResponse<Favorite> getFavoriteItems(@ApiParam(name = "page") @QueryParam("page") @DefaultValue("1") int page,
-                                                        @ApiParam(name = "per_page") @QueryParam("per_page") @DefaultValue("5") int perPage,
+    public PaginatedResponse<Favorite> getFavoriteItems(@ApiParam(name = "page") @QueryParam("page") @DefaultValue("1") @Min(1) int page,
+                                                        @ApiParam(name = "per_page") @QueryParam("per_page") @DefaultValue("5") @Min(1) int perPage,
                                                         @ApiParam(name = "type") @QueryParam("type") Optional<String> type,
                                                         @Context SearchUser searchUser) {
         return favoritesService.findFavoritesFor(searchUser, type, page, perPage);

diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/favorites/FavoritesService.java b/graylog2-server/src/main/java/org/graylog/plugins/views/favorites/FavoritesService.java
@@ -19,7 +19,7 @@
 import com.google.common.eventbus.EventBus;
 import com.google.common.eventbus.Subscribe;
 import com.mongodb.BasicDBObject;
-import com.mongodb.DuplicateKeyException;
+import com.mongodb.MongoException;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.model.Filters;
 import com.mongodb.client.model.Indexes;
@@ -30,7 +30,6 @@
 import org.graylog.plugins.views.startpage.recentActivities.RecentActivityEvent;
 import org.graylog.plugins.views.startpage.title.StartPageItemTitleRetriever;
 import org.graylog2.database.MongoCollections;
-import org.graylog2.database.PaginatedDbService;
 import org.graylog2.database.PaginatedList;
 import org.graylog2.database.utils.MongoUtils;
 import org.graylog2.rest.models.PaginatedResponse;
@@ -73,9 +72,11 @@ public PaginatedResponse<Favorite> findFavoritesFor(final SearchUser searchUser,
                         .map(title -> new Favorite(i, title))
                 )
                 .flatMap(Optional::stream)
+                .skip(perPage * Math.max(0L, page - 1))
+                .limit(perPage)
                 .toList();
 
-        return PaginatedResponse.create("favorites", new PaginatedList<>(PaginatedDbService.getPage(items, page, perPage), items.size(), page, perPage));
+        return PaginatedResponse.create("favorites", new PaginatedList<>(items, items.size(), page, perPage));
     }
 
     public void save(FavoritesForUserDTO favorite) {
@@ -129,8 +130,11 @@ public Optional<FavoritesForUserDTO> create(final FavoritesForUserDTO favorite)
         try {
             final var result = db.insertOne(favorite);
             return mongoUtils.getById(MongoUtils.insertedId(result));
-        } catch (DuplicateKeyException e) {
-            throw new IllegalStateException("Unable to create a Favorites collection, collection with this id already exists : " + favorite.id());
+        } catch (MongoException e) {
+            if (MongoUtils.isDuplicateKeyError(e)) {
+                throw new IllegalStateException("Unable to create favorites. Favorites with this id already exist: " + favorite.id());
+            }
+            throw e;
         }
     }
 

diff --git a/...n/java/org/graylog/plugins/views/search/validation/validators/UnknownFieldsValidator.java b/...n/java/org/graylog/plugins/views/search/validation/validators/UnknownFieldsValidator.java
@@ -16,6 +16,7 @@
  */
 package org.graylog.plugins.views.search.validation.validators;
 
+import jakarta.inject.Singleton;
 import org.graylog.plugins.views.search.engine.QueryPosition;
 import org.graylog.plugins.views.search.rest.MappedFieldTypeDTO;
 import org.graylog.plugins.views.search.validation.ParsedTerm;
@@ -26,8 +27,6 @@
 import org.graylog.plugins.views.search.validation.ValidationType;
 import org.graylog.plugins.views.search.validation.validators.util.UnknownFieldsListLimiter;
 
-import jakarta.inject.Singleton;
-
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -66,6 +65,7 @@ public List<ValidationMessage> validate(final ValidationContext context) {
     List<ParsedTerm> identifyUnknownFields(final Set<String> availableFields, final List<ParsedTerm> terms) {
         final Map<String, List<ParsedTerm>> groupedByField = terms.stream()
                 .filter(t -> !t.isDefaultField())
+                .filter(term -> !isParameter(term.getRealFieldName()))
                 .filter(term -> !SEARCHABLE_ES_FIELDS.contains(term.getRealFieldName()))
                 .filter(term -> !RESERVED_SETTABLE_FIELDS.contains(term.getRealFieldName()))
                 .filter(term -> !availableFields.contains(term.getRealFieldName()))
@@ -74,4 +74,8 @@ List<ParsedTerm> identifyUnknownFields(final Set<String> availableFields, final
 
         return unknownFieldsListLimiter.filterElementsContainingUsefulInformation(groupedByField);
     }
+
+    private boolean isParameter(final String term) {
+        return term.startsWith("$") && term.endsWith("$");
+    }
 }
diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/views/ViewService.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/views/ViewService.java
@@ -16,7 +16,7 @@
  */
 package org.graylog.plugins.views.search.views;
 
-import com.mongodb.DuplicateKeyException;
+import com.mongodb.MongoException;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.model.Filters;
 import com.mongodb.client.model.Sorts;
@@ -226,8 +226,11 @@ public ViewDTO save(ViewDTO viewDTO) {
         try {
             final var save = collection.insertOne(requirementsForView(viewDTO));
             return mongoUtils.getById(MongoUtils.insertedId(save)).orElseThrow(() -> new IllegalStateException("Unable to retrieve saved View!"));
-        } catch (DuplicateKeyException e) {
-            throw new IllegalStateException("Unable to save view, it already exists.");
+        } catch (MongoException e) {
+            if (MongoUtils.isDuplicateKeyError(e)) {
+                throw new IllegalStateException("Unable to save view, it already exists.");
+            }
+            throw e;
         }
     }
 

diff --git a/...erver/src/main/java/org/graylog/plugins/views/startpage/lastOpened/LastOpenedService.java b/...erver/src/main/java/org/graylog/plugins/views/startpage/lastOpened/LastOpenedService.java
@@ -19,7 +19,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.eventbus.EventBus;
 import com.google.common.eventbus.Subscribe;
-import com.mongodb.DuplicateKeyException;
+import com.mongodb.MongoException;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.model.Filters;
 import com.mongodb.client.model.Indexes;
@@ -59,8 +59,11 @@ Optional<LastOpenedForUserDTO> findForUser(final String userId) {
     public void create(final LastOpenedForUserDTO lastOpenedItems) {
         try {
             db.insertOne(lastOpenedItems);
-        } catch (DuplicateKeyException e) {
-            throw new IllegalStateException("Unable to create a last opened collection, collection with this id already exists : " + lastOpenedItems.id());
+        } catch (MongoException e) {
+            if (MongoUtils.isDuplicateKeyError(e)) {
+                throw new IllegalStateException("Unable to create record of last opened items. Record with this id already exists : " + lastOpenedItems.id());
+            }
+            throw e;
         }
     }
 

diff --git a/graylog2-server/src/main/java/org/graylog2/contentstream/db/ContentStreamUserSettings.java b/graylog2-server/src/main/java/org/graylog2/contentstream/db/ContentStreamUserSettings.java
@@ -19,6 +19,7 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.auto.value.AutoValue;
+import org.graylog2.database.MongoEntity;
 import org.mongojack.Id;
 import org.mongojack.ObjectId;
 
@@ -27,7 +28,7 @@
 import java.util.List;
 
 @AutoValue
-public abstract class ContentStreamUserSettings {
+public abstract class ContentStreamUserSettings implements MongoEntity {
 
     public static final String FIELD_USER_ID = "user_id";
     public static final String FIELD_CONTENT_ENABLED = "content_stream_enabled";

diff --git a/...erver/src/main/java/org/graylog2/contentstream/db/DBContentStreamUserSettingsService.java b/...erver/src/main/java/org/graylog2/contentstream/db/DBContentStreamUserSettingsService.java
@@ -16,42 +16,41 @@
  */
 package org.graylog2.contentstream.db;
 
-import org.bson.types.ObjectId;
-import org.graylog2.bindings.providers.MongoJackObjectMapperProvider;
-import org.graylog2.database.MongoConnection;
-import org.mongojack.DBQuery;
-import org.mongojack.JacksonDBCollection;
-
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.ReplaceOptions;
 import jakarta.inject.Inject;
+import org.graylog2.database.MongoCollections;
+import org.graylog2.database.utils.MongoUtils;
 
 import java.util.Optional;
 
-import static org.graylog2.telemetry.user.db.TelemetryUserSettingsDto.FIELD_USER_ID;
+import static org.graylog2.contentstream.db.ContentStreamUserSettings.FIELD_USER_ID;
 
 public class DBContentStreamUserSettingsService {
 
     public static final String COLLECTION_NAME = "content_stream_user_settings";
 
-    private final JacksonDBCollection<ContentStreamUserSettings, ObjectId> db;
+    private final MongoCollection<ContentStreamUserSettings> collection;
 
     @Inject
-    public DBContentStreamUserSettingsService(MongoConnection mongoConnection,
-                                              MongoJackObjectMapperProvider mapper) {
-        this.db = JacksonDBCollection.wrap(mongoConnection.getDatabase().getCollection(COLLECTION_NAME),
-                ContentStreamUserSettings.class,
-                ObjectId.class,
-                mapper.get());
+    public DBContentStreamUserSettingsService(MongoCollections mongoCollections) {
+        collection = mongoCollections.collection(COLLECTION_NAME, ContentStreamUserSettings.class);
     }
 
     public Optional<ContentStreamUserSettings> findByUserId(String userId) {
-        return Optional.ofNullable(db.findOne(DBQuery.is(FIELD_USER_ID, userId)));
+        return Optional.ofNullable(collection.find(Filters.eq(FIELD_USER_ID, userId)).first());
     }
 
     public void save(ContentStreamUserSettings dto) {
-        db.save(dto);
+        if (dto.id() == null) {
+            collection.insertOne(dto);
+        } else {
+            collection.replaceOne(MongoUtils.idEq(dto.id()), dto, new ReplaceOptions().upsert(true));
+        }
     }
 
     public void delete(String userId) {
-        db.remove(DBQuery.is(FIELD_USER_ID, userId));
+        collection.deleteMany(Filters.eq(FIELD_USER_ID, userId));
     }
 }