Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Load SPs by search during startup for faster boot. #6757

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Load SPs by search during startup for faster boot. #6757

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
de5533b
Load SPs in batches
michaelabuckley Feb 24, 2025
03c96d6
Changelog
michaelabuckley Feb 24, 2025
53816b9
Update test mock
michaelabuckley Feb 24, 2025
5ff2252
spotless
michaelabuckley Feb 25, 2025
6aa45b5
Merge branch 'refs/heads/master' into mb-faster-sp-loading
michaelabuckley Mar 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ...fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/8_2_0/6757-faster-sp-loading.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
---
type: perf
issue: 6757
title: "Speed up SearchParameter loading on server startup."
30 changes: 21 additions & 9 deletions ...archparam/src/main/java/ca/uhn/fhir/jpa/searchparam/registry/SearchParamRegistryImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,14 @@
import ca.uhn.fhir.rest.api.Constants;
import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum;
import ca.uhn.fhir.rest.api.server.IBundleProvider;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.rest.param.ReferenceOrListParam;
import ca.uhn.fhir.rest.param.ReferenceParam;
import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
import ca.uhn.fhir.rest.server.util.IndexedSearchParam;
import ca.uhn.fhir.rest.server.util.ResourceSearchParams;
import ca.uhn.fhir.util.SearchParameterUtil;
import ca.uhn.fhir.util.StopWatch;
import ca.uhn.fhir.util.TaskChunker;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
import jakarta.annotation.Nonnull;
Expand All @@ -66,6 +68,7 @@
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static ca.uhn.fhir.rest.server.util.ISearchParamRegistry.isAllowedForContext;
import static org.apache.commons.lang3.StringUtils.isBlank;
Expand All @@ -82,6 +85,7 @@ public class SearchParamRegistryImpl
private static final Logger ourLog = LoggerFactory.getLogger(SearchParamRegistryImpl.class);
public static final int MAX_MANAGED_PARAM_COUNT = 10000;
private static final long REFRESH_INTERVAL = DateUtils.MILLIS_PER_MINUTE;
private static final int QUERY_CHUNK_SIZE = 800; // Max number of bind params

private JpaSearchParamCache myJpaSearchParamCache;

Expand Down Expand Up @@ -459,15 +463,23 @@ private String unqualified(List<IIdType> theIds) {

@Override
public void handleInit(Collection<IIdType> theResourceIds) {
StopWatch sw = new StopWatch();
List<IBaseResource> searchParams = new ArrayList<>();
for (IIdType id : theResourceIds) {
try {
IBaseResource searchParam = mySearchParamProvider.read(id);
searchParams.add(searchParam);
} catch (ResourceNotFoundException e) {
ourLog.warn("SearchParameter {} not found. Excluding from list of active search params.", id);
}
}

Stream<ReferenceParam> paramStream = theResourceIds.stream().map(ReferenceParam::new);

TaskChunker.chunk(paramStream, QUERY_CHUNK_SIZE)
.map(nextReferenceChunk -> {
ReferenceOrListParam accumulator = new ReferenceOrListParam();
nextReferenceChunk.forEach(accumulator::addOr);
SearchParameterMap params =
SearchParameterMap.newSynchronous().add("_id", accumulator);
return mySearchParamProvider.search(params).getAllResources();
})
.forEach(searchParams::addAll);

ourLog.debug("Read {} SearchParameters from database in {}ms", theResourceIds.size(), sw.getMillis());

initializeActiveSearchParams(searchParams);
}

Expand Down
25 changes: 12 additions & 13 deletions ...param/src/test/java/ca/uhn/fhir/jpa/searchparam/registry/SearchParamRegistryImplTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import ca.uhn.fhir.model.primitive.IdDt;
import ca.uhn.fhir.rest.server.SimpleBundleProvider;
import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
import ca.uhn.fhir.rest.server.util.ResourceSearchParams;
import ca.uhn.fhir.util.HapiExtensions;
Expand Down Expand Up @@ -52,6 +51,7 @@
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;

import static ca.uhn.fhir.rest.server.util.ISearchParamRegistry.SearchParamLookupContextEnum.ALL;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
Expand Down Expand Up @@ -141,20 +141,19 @@ public void after() {

@Test
void handleInit() {
assertEquals(31, mySearchParamRegistry.getActiveSearchParams("Patient", null).size());
assertEquals(31, mySearchParamRegistry.getActiveSearchParams("Patient", ALL).size());

IdDt idBad = new IdDt("SearchParameter/bad");
when(mySearchParamProvider.read(idBad)).thenThrow(new ResourceNotFoundException("id bad"));

IdDt idGood = new IdDt("SearchParameter/good");
SearchParameter goodSearchParam = buildSearchParameter(Enumerations.PublicationStatus.ACTIVE);
when(mySearchParamProvider.read(idGood)).thenReturn(goodSearchParam);
when(mySearchParamProvider.search(any())).thenReturn(new SimpleBundleProvider(goodSearchParam));

List<IIdType> idList = new ArrayList<>();
idList.add(idBad);
idList.add(idGood);
mySearchParamRegistry.handleInit(idList);
assertEquals(32, mySearchParamRegistry.getActiveSearchParams("Patient", null).size());
assertEquals(32, mySearchParamRegistry.getActiveSearchParams("Patient", ALL).size());
}

@Test
Expand Down Expand Up @@ -229,7 +228,7 @@ public void testSearchParamUpdate() {
}

private void assertPatientSearchParamSize(int theExpectedSize) {
assertEquals(theExpectedSize, mySearchParamRegistry.getActiveSearchParams("Patient", null).size());
assertEquals(theExpectedSize, mySearchParamRegistry.getActiveSearchParams("Patient", ALL).size());
}

private void assertResult(ResourceChangeResult theResult, long theExpectedAdded, long theExpectedUpdated, long theExpectedRemoved) {
Expand All @@ -256,19 +255,19 @@ private void assertDbNotCalled() {

@Test
public void testGetActiveUniqueSearchParams_Empty() {
assertThat(mySearchParamRegistry.getActiveComboSearchParams("Patient", null)).isEmpty();
assertThat(mySearchParamRegistry.getActiveComboSearchParams("Patient", ALL)).isEmpty();
}

@Test
public void testGetActiveSearchParamByUrl_whenSPExists_returnsActiveSp() {
RuntimeSearchParam patientLanguageSp = mySearchParamRegistry.getActiveSearchParamByUrl("SearchParameter/Patient-language", null);
RuntimeSearchParam patientLanguageSp = mySearchParamRegistry.getActiveSearchParamByUrl("SearchParameter/Patient-language", ALL);
assertNotNull(patientLanguageSp);
assertEquals(patientLanguageSp.getId().getIdPart(), "Patient-language");
}

@Test
public void testGetActiveSearchParamByUrl_whenSPNotExist_returnsNull() {
RuntimeSearchParam nonExistingSp = mySearchParamRegistry.getActiveSearchParamByUrl("SearchParameter/nonExistingSp", null);
RuntimeSearchParam nonExistingSp = mySearchParamRegistry.getActiveSearchParamByUrl("SearchParameter/nonExistingSp", ALL);
assertNull(nonExistingSp);
}

Expand All @@ -287,7 +286,7 @@ public void testGetActiveSearchParamsRetries() {

assertFalse(retried.get());
mySearchParamRegistry.forceRefresh();
ResourceSearchParams activeSearchParams = mySearchParamRegistry.getActiveSearchParams("Patient", null);
ResourceSearchParams activeSearchParams = mySearchParamRegistry.getActiveSearchParams("Patient", ALL);
assertTrue(retried.get());
assertEquals(ourBuiltInSearchParams.getSearchParamMap("Patient").size(), activeSearchParams.size());
}
Expand All @@ -300,7 +299,7 @@ public void testAddActiveSearchparam() {
resetDatabaseToOrigSearchParamsPlusNewOneWithStatus(Enumerations.PublicationStatus.ACTIVE);

mySearchParamRegistry.forceRefresh();
ResourceSearchParams activeSearchParams = mySearchParamRegistry.getActiveSearchParams("Patient", null);
ResourceSearchParams activeSearchParams = mySearchParamRegistry.getActiveSearchParams("Patient", ALL);

RuntimeSearchParam converted = activeSearchParams.get("foo");
assertNotNull(converted);
Expand Down Expand Up @@ -335,7 +334,7 @@ public void testUpliftRefchains() {

mySearchParamRegistry.forceRefresh();

RuntimeSearchParam canonicalSp = mySearchParamRegistry.getRuntimeSearchParam("Encounter", "subject", null);
RuntimeSearchParam canonicalSp = mySearchParamRegistry.getRuntimeSearchParam("Encounter", "subject", ALL);
assertEquals("Modified Subject", canonicalSp.getDescription());
assertTrue(canonicalSp.hasUpliftRefchain("name1"));
assertFalse(canonicalSp.hasUpliftRefchain("name99"));
Expand All @@ -344,7 +343,7 @@ public void testUpliftRefchains() {

private List<ResourceTable> resetDatabaseToOrigSearchParamsPlusNewOneWithStatus(Enumerations.PublicationStatus theStatus) {
// Add a new search parameter entity
List<ResourceTable> newEntities = new ArrayList(ourEntities);
List<ResourceTable> newEntities = new ArrayList<>(ourEntities);
newEntities.add(createEntity(++ourLastId, 1));
resetMock(theStatus, newEntities);
return newEntities;
Expand Down
Loading