knife fix when the rows in wild search exceeds 10k.

Signed-off-by: talktovikas <[email protected]>
chef · Dec 12, 2024 · 6ca1f97 · 6ca1f97
1 parent 599127a
commit 6ca1f97
Show file tree

Hide file tree

Showing 6 changed files with 39 additions and 0 deletions.
diff --git a/omnibus/files/server-ctl-cookbooks/infra-server/templates/default/oc_erchef.config.erb b/omnibus/files/server-ctl-cookbooks/infra-server/templates/default/oc_erchef.config.erb
@@ -194,6 +194,7 @@
             {timeout, <%= @solr_timeout %>},
             {init_count, <%= @solr_http_init_count %>},
             {max_count, <%= @solr_http_max_count %>},
+            {track_total_hits, <%= @solr_track_total_hits %>},
             {cull_interval, <%= @solr_http_cull_interval %>},
             {max_age, <%= @solr_http_max_age %>},
             {max_connection_duration, <%= @solr_http_max_connection_duration %>},

diff --git a/src/oc_erchef/apps/chef_index/src/chef_index_query.erl b/src/oc_erchef/apps/chef_index/src/chef_index_query.erl
@@ -30,6 +30,7 @@ from_params(Provider, ObjType, QueryString, Start, Rows) ->
                      search_provider = Provider,
                      start = decode({nonneg_int, "start"}, Start, 0),
                      rows = decode({nonneg_int, "rows"}, Rows, 1000),
+                     track_total_hits = envy:get(chef_index, track_total_hits, false, boolean),
                      sort = "X_CHEF_id_CHEF_X asc",
                      index = index_type(ObjType)}.
 

diff --git a/src/oc_erchef/apps/chef_index/src/chef_opensearch.erl b/src/oc_erchef/apps/chef_index/src/chef_opensearch.erl
@@ -103,21 +103,54 @@ query_body(#chef_solr_query{
               query_string = Query,
               filter_query = undefined,
               start = Start,
+              track_total_hits = true,
               rows = Rows}) ->
     jiffy:encode({[{fields_tag(), <<"_id">>},
                    {<<"from">>, Start},
                    {<<"size">>, Rows},
+                   {<<"track_total_hits">>,true},
                    {<<"query">>, {[query_string_query_ejson(Query)]}}
                   ]});
+query_body(#chef_solr_query{
+    query_string = Query,
+    filter_query = undefined,
+    start = Start,
+    track_total_hits = false,
+    rows = Rows}) ->
+    jiffy:encode({[{fields_tag(), <<"_id">>},
+                    {<<"from">>, Start},
+                    {<<"size">>, Rows},
+                    {<<"query">>, {[query_string_query_ejson(Query)]}}
+                ]});
+
+query_body(#chef_solr_query{
+    query_string = Query,
+    filter_query = FilterQuery,
+    start = Start,
+    track_total_hits = false,
+    rows = Rows}) ->
+chef_index_query:assert_org_id_filter(FilterQuery),
+jiffy:encode({[{ fields_tag(), <<"_id">>},
+    {<<"from">>, Start},
+    {<<"size">>, Rows},
+    {<<"sort">>, [{[{<<"X_CHEF_id_CHEF_X">>, {[{<<"order">>, <<"asc">>}]}}]}]},
+    {<<"query">>, {[
+        {<<"bool">>,{[
+            {<<"must">>, {[query_string_query_ejson(Query)]}},
+            {<<"filter">>, {[query_string_query_ejson(FilterQuery)]}}
+        ]}}]}
+    }]});
 query_body(#chef_solr_query{
               query_string = Query,
               filter_query = FilterQuery,
               start = Start,
+              track_total_hits = true,
               rows = Rows}) ->
     chef_index_query:assert_org_id_filter(FilterQuery),
     jiffy:encode({[{ fields_tag(), <<"_id">>},
         {<<"from">>, Start},
         {<<"size">>, Rows},
+        {<<"track_total_hits">>,true},
         {<<"sort">>, [{[{<<"X_CHEF_id_CHEF_X">>, {[{<<"order">>, <<"asc">>}]}}]}]},
         {<<"query">>, {[
             {<<"bool">>,{[

diff --git a/src/oc_erchef/apps/chef_index/test/chef_index_query_tests.erl b/src/oc_erchef/apps/chef_index/test/chef_index_query_tests.erl
@@ -40,6 +40,7 @@ query_from_params_test_() ->
             sort = "X_CHEF_id_CHEF_X asc",
             start = 2,
             rows = 5,
+            track_total_hits=false,
             index = node},
           ?assertEqual(Expect, Query)
       end},
@@ -57,6 +58,7 @@ query_from_params_test_() ->
             sort = "X_CHEF_id_CHEF_X asc",
             start = 0,
             rows = 1000,
+            track_total_hits=false,
             index = role},
           ?assertEqual(Expect, Query)
       end},

diff --git a/src/oc_erchef/habitat/config/sys.config b/src/oc_erchef/habitat/config/sys.config
@@ -173,6 +173,7 @@
         {search_batch_max_wait, 10},
         {reindex_sleep_min_ms, 500},
         {reindex_sleep_max_ms, 2000},
+        {track_total_hits,false},
         {reindex_item_retries, 3},
         {solr_elasticsearch_major_version, 5},
         {solr_service, [

diff --git a/src/oc_erchef/include/chef_solr.hrl b/src/oc_erchef/include/chef_solr.hrl
@@ -21,6 +21,7 @@
           search_provider = solr :: 'solr' | 'elasticsearch' | 'opensearch',
           start :: integer() | undefined,
           rows :: integer()  | undefined,
+          track_total_hits :: boolean() |undefined,
           sort :: string()   | undefined,
           index :: 'node'
                  | 'role'