diff --git a/store/CHANGELOG.md b/store/CHANGELOG.md
index cdd7bb1c59fa..84b2a0d35eb3 100644
--- a/store/CHANGELOG.md
+++ b/store/CHANGELOG.md
@@ -23,6 +23,12 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 # Changelog
 
+## [Unreleased]
+
+### Bug Fixes
+
+* (store) [#20425](https://github.com/cosmos/cosmos-sdk/pull/20425) Fix nil pointer panic when query historical state where a new store don't exist.
+
 ## v1.1.0 (March 20, 2024)
 
 ### Improvements
diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go
index cfcc5f47b8b2..dcd418ccb09c 100644
--- a/store/rootmulti/store.go
+++ b/store/rootmulti/store.go
@@ -607,6 +607,10 @@ func (rs *Store) CacheMultiStoreWithVersion(version int64) (types.CacheMultiStor
 				if storeInfos[key.Name()] {
 					return nil, err
 				}
+
+				// If the store donesn't exist at this version, create a dummy one to prevent
+				// nil pointer panic in newer query APIs.
+				cacheStore = dbadapter.Store{DB: dbm.NewMemDB()}
 			}
 
 		default:
diff --git a/store/rootmulti/store_test.go b/store/rootmulti/store_test.go
index 2702f3e08623..ebdd657de41f 100644
--- a/store/rootmulti/store_test.go
+++ b/store/rootmulti/store_test.go
@@ -100,16 +100,19 @@ func TestCacheMultiStoreWithVersion(t *testing.T) {
 	require.Equal(t, kvStore.Get(k), v)
 
 	// add new module stores (store4 and store5) to multi stores and commit
-	ms.MountStoreWithDB(types.NewKVStoreKey("store4"), types.StoreTypeIAVL, nil)
-	ms.MountStoreWithDB(types.NewKVStoreKey("store5"), types.StoreTypeIAVL, nil)
+	key4, key5 := types.NewKVStoreKey("store4"), types.NewKVStoreKey("store5")
+	ms.MountStoreWithDB(key4, types.StoreTypeIAVL, nil)
+	ms.MountStoreWithDB(key5, types.StoreTypeIAVL, nil)
 	err = ms.LoadLatestVersionAndUpgrade(&types.StoreUpgrades{Added: []string{"store4", "store5"}})
 	require.NoError(t, err)
 	ms.Commit()
 
 	// cache multistore of version before adding store4 should works
-	_, err = ms.CacheMultiStoreWithVersion(1)
+	cms2, err := ms.CacheMultiStoreWithVersion(1)
 	require.NoError(t, err)
 
+	require.Empty(t, cms2.GetKVStore(key4).Get([]byte("key")))
+
 	// require we cannot commit (write) to a cache-versioned multi-store
 	require.Panics(t, func() {
 		kvStore.Set(k, []byte("newValue"))