Merge branch 'master' into sidebar-page-links

changelog/unreleased/issue-20592.toml

@@ -0,0 +1,5 @@
+type = "fixed"
+message = "Fix that non visible items are selected in index sets table when changing field type"
+
+issues = ["20592"]
+pulls = ["20616"]

graylog2-server/src/main/java/org/graylog/events/processor/DBEventProcessorStateService.java

@@ -19,10 +19,10 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
 import com.mongodb.BasicDBObject;
+import jakarta.inject.Inject;
 import org.bson.types.ObjectId;
 import org.graylog2.bindings.providers.MongoJackObjectMapperProvider;
 import org.graylog2.database.MongoConnection;
-import org.graylog2.database.MongoDBUpsertRetryer;
 import org.joda.time.DateTime;
 import org.mongojack.DBQuery;
 import org.mongojack.DBUpdate;
@@ -32,8 +32,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import jakarta.inject.Inject;
-
 import java.util.Optional;
 import java.util.Set;
 
@@ -140,15 +138,15 @@ public Optional<EventProcessorStateDto> setState(String eventDefinitionId,
                 .addOperation("$min", FIELD_MIN_PROCESSED_TIMESTAMP, updateValue(minProcessedTimestamp))
                 .addOperation("$max", FIELD_MAX_PROCESSED_TIMESTAMP, updateValue(maxProcessedTimestamp));
 
-        return Optional.ofNullable(MongoDBUpsertRetryer.run(() -> db.findAndModify(
+        return Optional.ofNullable(db.findAndModify(
                 // We have a unique index on the eventDefinitionId so this query is enough
                 DBQuery.is(FIELD_EVENT_DEFINITION_ID, eventDefinitionId),
                 null,
                 null,
                 false,
                 update,
                 true, // We want to return the updated document to the caller
-                true)));
+                true));
     }
 
     /**

graylog2-server/src/main/java/org/graylog2/database/MongoDBUpsertRetryer.java

@@ -22,8 +22,8 @@
 import com.github.rholder.retry.Retryer;
 import com.github.rholder.retry.RetryerBuilder;
 import com.github.rholder.retry.StopStrategies;
-import com.mongodb.DuplicateKeyException;
-import com.mongodb.MongoCommandException;
+import com.mongodb.MongoException;
+import org.graylog2.database.utils.MongoUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,16 +33,20 @@
 /**
  * MongoDB upsert requests can fail when they are creating a new entry concurrently.
  * https://jira.mongodb.org/browse/SERVER-14322
- * This helper can be used to retry upserts if they throw a {@link DuplicateKeyException}
+ * This helper can be used to retry upserts if they throw a duplicate key error
+ *
+ * @deprecated This class should not be used anymore. It was intended for usage with MongoDB versions < 4.2 which are
+ * not supported for usage with Graylog anymore. Recent versions of MongoDB implement server-side retries for certain
+ * upsert scenarios.
  */
+@Deprecated(forRemoval = true)
 public class MongoDBUpsertRetryer {
     private static final Logger LOG = LoggerFactory.getLogger(MongoDBUpsertRetryer.class);
 
     public static <T> T run(Callable<T> c) {
         final Retryer<T> retryer = RetryerBuilder.<T>newBuilder()
-                .retryIfException(t -> t instanceof DuplicateKeyException && ((DuplicateKeyException) t).getErrorCode() == 11000 ||
-                        t instanceof MongoCommandException && ((MongoCommandException) t).getErrorCode() == 11000)
-                .withStopStrategy(StopStrategies.stopAfterAttempt(2))
+                .retryIfException(t -> t instanceof MongoException e && MongoUtils.isDuplicateKeyError(e))
+                .withStopStrategy(StopStrategies.stopAfterAttempt(3))
                 .withRetryListener(new RetryListener() {
                     @Override
                     public <V> void onRetry(Attempt<V> attempt) {
@@ -57,8 +61,8 @@ public <V> void onRetry(Attempt<V> attempt) {
         } catch (ExecutionException e) {
             throw new RuntimeException(e.getCause());
         } catch (RetryException e) {
-            if (e.getCause() instanceof DuplicateKeyException) {
-                throw (DuplicateKeyException) e.getCause();
+            if (e.getCause() instanceof RuntimeException re) {
+                throw re;
             }
             throw new RuntimeException(e.getCause());
         }

graylog2-server/src/main/java/org/graylog2/indexer/fieldtypes/IndexFieldTypesService.java

@@ -28,7 +28,6 @@
 import org.bson.types.ObjectId;
 import org.graylog2.bindings.providers.MongoJackObjectMapperProvider;
 import org.graylog2.database.MongoConnection;
-import org.graylog2.database.MongoDBUpsertRetryer;
 import org.mongojack.DBQuery;
 import org.mongojack.JacksonDBCollection;
 import org.mongojack.WriteResult;
@@ -132,15 +131,15 @@ public IndexFieldTypesDTO save(IndexFieldTypesDTO dto) {
     }
 
     public Optional<IndexFieldTypesDTO> upsert(IndexFieldTypesDTO dto) {
-        final WriteResult<IndexFieldTypesDTO, ObjectId> update = MongoDBUpsertRetryer.run(() -> db.update(
+        final WriteResult<IndexFieldTypesDTO, ObjectId> update = db.update(
                 DBQuery.and(
                         DBQuery.is(FIELD_INDEX_NAME, dto.indexName()),
                         DBQuery.is(FIELD_INDEX_SET_ID, dto.indexSetId())
                 ),
                 dto,
                 true,
                 false
-        ));
+        );
 
         final Object upsertedId = update.getUpsertedId();
         if (upsertedId instanceof ObjectId) {

graylog2-server/src/main/java/org/graylog2/system/processing/DBProcessingStatusService.java

@@ -21,11 +21,12 @@
 import com.google.common.collect.ImmutableList;
 import com.mongodb.BasicDBObject;
 import com.mongodb.MongoException;
+import jakarta.inject.Inject;
+import jakarta.inject.Named;
 import org.bson.types.ObjectId;
 import org.graylog.scheduler.clock.JobSchedulerClock;
 import org.graylog2.bindings.providers.MongoJackObjectMapperProvider;
 import org.graylog2.database.MongoConnection;
-import org.graylog2.database.MongoDBUpsertRetryer;
 import org.graylog2.plugin.BaseConfiguration;
 import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
 import org.graylog2.plugin.system.NodeId;
@@ -36,9 +37,6 @@
 import org.mongojack.DBSort;
 import org.mongojack.JacksonDBCollection;
 
-import jakarta.inject.Inject;
-import jakarta.inject.Named;
-
 import java.util.List;
 import java.util.Optional;
 
@@ -132,14 +130,14 @@ ProcessingStatusDto save(ProcessingStatusRecorder processingStatusRecorder, Date
         // TODO: Using a timestamp provided by the node for "updated_at" can be bad if the node clock is skewed.
         //       Ideally we would use MongoDB's "$currentDate" but there doesn't seem to be a way to use that
         //       with mongojack.
-        return MongoDBUpsertRetryer.run(() -> db.findAndModify(
+        return db.findAndModify(
                 DBQuery.is(ProcessingStatusDto.FIELD_NODE_ID, nodeId),
                 null,
                 null,
                 false,
                 ProcessingStatusDto.of(nodeId, processingStatusRecorder, updatedAt, baseConfiguration.isMessageJournalEnabled()),
                 true, // We want to return the updated document to the caller
-                true));
+                true);
     }
 
     /**

graylog2-server/src/main/java/org/graylog2/users/RoleServiceImpl.java

@@ -28,7 +28,6 @@
 import org.bson.types.ObjectId;
 import org.graylog2.bindings.providers.MongoJackObjectMapperProvider;
 import org.graylog2.database.MongoConnection;
-import org.graylog2.database.MongoDBUpsertRetryer;
 import org.graylog2.database.NotFoundException;
 import org.graylog2.plugin.database.ValidationException;
 import org.graylog2.shared.security.Permissions;
@@ -204,8 +203,7 @@ public RoleImpl save(Role role1) throws ValidationException {
         if (!violations.isEmpty()) {
             throw new ValidationException("Validation failed.", violations.toString());
         }
-        return MongoDBUpsertRetryer.run(() ->
-                dbCollection.findAndModify(is(NAME_LOWER, role.nameLower()), null, null, false, role, true, true));
+        return dbCollection.findAndModify(is(NAME_LOWER, role.nameLower()), null, null, false, role, true, true);
     }
 
     @Override

graylog2-web-interface/package.json

@@ -55,7 +55,7 @@
     "@reduxjs/toolkit": "^2.2.0",
     "@tanstack/query-sync-storage-persister": "^4.33.0",
     "@tanstack/react-query-persist-client": "^4.33.0",
-    "ace-builds": "1.36.1",
+    "ace-builds": "1.36.2",
     "bootstrap": "3.4.1",
     "bson-objectid": "^2.0.3",
     "chroma-js": "^2.0.3",

graylog2-web-interface/packages/graylog-web-plugin/package.json

@@ -48,7 +48,7 @@
     "react": "18.3.1",
     "react-dom": "18.2.0",
     "react-router-bootstrap": "0.26.3",
-    "react-router-dom": "6.26.2",
+    "react-router-dom": "6.27.0",
     "reflux": "0.2.13",
     "styled-components": "6.1.1",
     "typescript": "5.6.3",

graylog2-web-interface/packages/stylelint-config-graylog/package.json

@@ -14,7 +14,7 @@
   "license": "SSPL-1.0",
   "dependencies": {
     "postcss-styled-syntax": "0.6.4",
-    "stylelint": "16.9.0",
+    "stylelint": "16.10.0",
     "stylelint-config-recommended": "14.0.1",
     "stylelint-config-standard": "36.0.1",
     "stylelint-config-styled-components": "0.1.1"

graylog2-web-interface/src/components/indices/helpers/isIndexFieldTypeChangeAllowed.ts

@@ -18,6 +18,7 @@
 import type { IndexSet } from 'stores/indices/IndexSetsStore';
 
 const TEMPLATE_TYPES = ['failures', 'events', 'illuminate_content'];
-const isIndexFieldTypeChangeAllowed = (indexSet: IndexSet) => !TEMPLATE_TYPES.includes(indexSet?.index_template_type);
+export const isTemplateTypeAllowsFieldTypeChang = (type: string) => !TEMPLATE_TYPES.includes(type);
+const isIndexFieldTypeChangeAllowed = (indexSet: IndexSet) => isTemplateTypeAllowsFieldTypeChang(indexSet?.index_template_type);
 
 export default isIndexFieldTypeChangeAllowed;

graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/ChangeFieldType.test.tsx

@@ -58,7 +58,7 @@ describe('ChangeFieldType', () => {
   useViewsPlugin();
 
   beforeAll(() => {
-    asMock(useInitialSelection).mockReturnValue(['id-1', 'id-2']);
+    asMock(useInitialSelection).mockReturnValue({ list: ['id-1', 'id-2'], isLoading: false });
     asMock(useFieldTypeUsages).mockReturnValue(paginatedFieldUsage);
   });
 

graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/ChangeFieldType.tsx

@@ -34,13 +34,14 @@ const ChangeFieldType = ({
     onClose();
   }, [onClose]);
 
-  const initialSelection = useInitialSelection();
+  const { list, isLoading } = useInitialSelection();
 
   return show ? (
-    <ChangeFieldTypeModal initialSelectedIndexSets={initialSelection}
+    <ChangeFieldTypeModal initialSelectedIndexSets={list}
                           onClose={handleOnClose}
                           initialData={{ fieldName: field }}
-                          show />
+                          show
+                          initialSelectionDataLoaded={!isLoading} />
   ) : null;
 };
 

graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/ChangeFieldTypeModal.tsx

@@ -63,7 +63,8 @@ type Props = {
   initialData?: {
     type?: string,
     fieldName?: string
-  }
+  },
+  initialSelectionDataLoaded?: boolean,
 }
 
 const FailureStreamLink = () => {
@@ -86,6 +87,7 @@ const ChangeFieldTypeModal = ({
   showSelectionTable,
   showFieldSelect,
   initialData,
+  initialSelectionDataLoaded,
 }: Props) => {
   const [{ fieldName, type }, setModalData] = useState<{ fieldName?: string, type?: string }>(initialData);
   const { data: { fieldTypes }, isLoading: isLoadingFieldTypes } = useFieldTypesForMappings();
@@ -185,7 +187,7 @@ const ChangeFieldTypeModal = ({
                         inputProps={{ 'aria-label': 'Select Field Type For Field' }}
                         required />
         </Input>
-        {showSelectionTable && (
+        {showSelectionTable && initialSelectionDataLoaded && (
           <>
             <StyledLabel>Select Targeted Index Sets</StyledLabel>
             <p>
@@ -214,6 +216,7 @@ ChangeFieldTypeModal.defaultProps = {
   onSubmitCallback: undefined,
   showFieldSelect: false,
   initialData: { fieldName: undefined, type: undefined },
+  initialSelectionDataLoaded: true,
 };
 
 export default ChangeFieldTypeModal;

graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/hooks/useAllIndexSetIds.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+import { useQuery } from '@tanstack/react-query';
+
+import UserNotification from 'util/UserNotification';
+import fetch from 'logic/rest/FetchProvider';
+import { qualifyUrl } from 'util/URLUtils';
+
+export type SimpleIndexSet = { id: string, type: string };
+type IndexSetsState = Array<SimpleIndexSet>;
+const INITIAL_DATA: IndexSetsState = [];
+
+const url = qualifyUrl('system/indices/index_sets/types/index_sets_with_field_type_change_support');
+
+const fetchAllIndexSetIds = async (streams: Array<string>): Promise<IndexSetsState> => fetch<Array<{index_set_id: string, type: string }>>('POST', url, streams?.length ? streams : undefined).then(
+  (elements) => {
+    const list: Array<SimpleIndexSet> = elements.map((element) => ({
+      id: element.index_set_id,
+      type: element.type,
+    }));
+
+    return list;
+  },
+);
+
+const useAllIndexSetIds = (streams: Array<string>): {
+  data: IndexSetsState,
+  isLoading: boolean,
+  refetch: () => void,
+} => {
+  const { data, isLoading, refetch } = useQuery(
+    ['allIndexSetIds', ...streams],
+    () => fetchAllIndexSetIds(streams),
+    {
+      onError: (errorThrown) => {
+        UserNotification.error(`Loading index sets with field type change support failed with status: ${errorThrown}`,
+          'Could not load index sets with field type change support');
+      },
+    },
+  );
+
+  return ({
+    data: data ?? INITIAL_DATA,
+    isLoading,
+    refetch,
+  });
+};
+
+export default useAllIndexSetIds;

graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/hooks/useInitialSelection.tsx

@@ -16,35 +16,19 @@
  */
 import { useMemo } from 'react';
 
-import { useStore } from 'stores/connect';
-import type { Stream } from 'views/stores/StreamsStore';
-import { StreamsStore } from 'views/stores/StreamsStore';
 import useCurrentStream from 'views/logic/fieldactions/ChangeFieldType/hooks/useCurrentStream';
-import type { IndexSet } from 'stores/indices/IndexSetsStore';
-import isIndexFieldTypeChangeAllowed from 'components/indices/helpers/isIndexFieldTypeChangeAllowed';
-import useIndexSetsList from 'components/indices/hooks/useIndexSetsList';
-
-const streamsMapper = ({ streams }) => streams.map((stream: Stream) => ({ indexSet: stream.index_set_id, id: stream.id }));
-
-const indexSetsMapper = (indexSets: Array<IndexSet>): Record<string, IndexSet> => {
-  if (!indexSets) return null;
-
-  return Object.fromEntries(indexSets.map((indexSet) => ([indexSet.id, indexSet])));
-};
+import {
+  isTemplateTypeAllowsFieldTypeChang,
+} from 'components/indices/helpers/isIndexFieldTypeChangeAllowed';
+import useAllIndexSetIds from 'views/logic/fieldactions/ChangeFieldType/hooks/useAllIndexSetIds';
 
 const useInitialSelection = () => {
   const currentStreams = useCurrentStream();
+  const { data, isLoading } = useAllIndexSetIds(currentStreams);
 
-  const availableStreams: Array<{ indexSet: string, id: string }> = useStore(StreamsStore, streamsMapper);
-  const { data } = useIndexSetsList();
-  const indexSets = useMemo(() => indexSetsMapper(data.indexSets), [data.indexSets]);
-
-  return useMemo(() => {
-    const currentStreamSet = new Set(currentStreams);
-    const filterFn = currentStreamSet.size > 0 ? ({ id, indexSet }) => currentStreamSet.has(id) && isIndexFieldTypeChangeAllowed(indexSets[indexSet]) : () => true;
+  const list = useMemo(() => data.filter(({ type }) => isTemplateTypeAllowsFieldTypeChang(type)).map(({ id }) => id), [data]);
 
-    return indexSets ? availableStreams.filter(filterFn).map(({ indexSet }) => indexSet) : [];
-  }, [availableStreams, currentStreams, indexSets]);
+  return ({ list, isLoading });
 };
 
 export default useInitialSelection;