diff --git a/test-tool-sample-app/.gitignore b/test-tool-sample-app/.gitignore
index e53e5922..64d7aaf8 100644
--- a/test-tool-sample-app/.gitignore
+++ b/test-tool-sample-app/.gitignore
@@ -26,8 +26,10 @@ appsettings.json
 _storage_emulator
 
 # Local data
+.localConfigs.testTool
 .localConfigs
 .notification.localstore.json
+.notification.testtoolstore.json
 
 # production
 /build
diff --git a/test-tool-sample-app/.localConfigs.testTool b/test-tool-sample-app/.localConfigs.testTool
new file mode 100644
index 00000000..a3981c2a
--- /dev/null
+++ b/test-tool-sample-app/.localConfigs.testTool
@@ -0,0 +1,4 @@
+# A gitignored place holder file for local runtime configurations when debug in test tool
+BOT_ID=
+BOT_PASSWORD=
+TEAMSFX_NOTIFICATION_STORE_FILENAME=.notification.testtoolstore.json
\ No newline at end of file
diff --git a/test-tool-sample-app/.vscode/launch.json b/test-tool-sample-app/.vscode/launch.json
index 8fa2df20..1219b39f 100644
--- a/test-tool-sample-app/.vscode/launch.json
+++ b/test-tool-sample-app/.vscode/launch.json
@@ -7,7 +7,7 @@
             "request": "launch",
             "url": "https://teams.microsoft.com/l/app/${{TEAMS_APP_ID}}?installAppPackage=true&webjoin=true&${account-hint}",
             "presentation": {
-                "group": "2-remote",
+                "group": "3-remote",
                 "order": 1
             },
             "internalConsoleOptions": "neverOpen"
@@ -18,7 +18,7 @@
             "request": "launch",
             "url": "https://teams.microsoft.com/l/app/${{TEAMS_APP_ID}}?installAppPackage=true&webjoin=true&${account-hint}",
             "presentation": {
-                "group": "2-remote",
+                "group": "3-remote",
                 "order": 2
             },
             "internalConsoleOptions": "neverOpen"
@@ -73,7 +73,7 @@
             ],
             "preLaunchTask": "Start Teams App Locally",
             "presentation": {
-                "group": "1-local",
+                "group": "2-local",
                 "order": 1
             },
             "stopAll": true
@@ -86,7 +86,7 @@
             ],
             "preLaunchTask": "Start Teams App Locally",
             "presentation": {
-                "group": "1-local",
+                "group": "2-local",
                 "order": 2
             },
             "stopAll": true
@@ -98,8 +98,8 @@
             ],
             "preLaunchTask": "Start Teams App (Test Tool)",
             "presentation": {
-                "group": "0-testtool",
-                "order": 2
+                "group": "1-local",
+                "order": 1
             },
             "stopAll": true
         },
diff --git a/test-tool-sample-app/.vscode/tasks.json b/test-tool-sample-app/.vscode/tasks.json
index 1a3b666f..1c3e241f 100644
--- a/test-tool-sample-app/.vscode/tasks.json
+++ b/test-tool-sample-app/.vscode/tasks.json
@@ -9,7 +9,7 @@
             "dependsOn": [
                 "Validate prerequisites (Test Tool)",
                 "Deploy (Test Tool)",
-                "Start application for Test Tool",
+                "Start application (Test Tool)",
                 "Start Test Tool",
             ],
             "dependsOrder": "sequence"
@@ -43,7 +43,7 @@
             }
         },
         {
-            "label": "Start application for Test Tool",
+            "label": "Start application (Test Tool)",
             "type": "shell",
             "command": "npm run dev:teamsfx:testtool",
             "isBackground": true,
diff --git a/test-tool-sample-app/env/.env.testtool b/test-tool-sample-app/env/.env.testtool
index 5b9697b5..1ba3ab0b 100644
--- a/test-tool-sample-app/env/.env.testtool
+++ b/test-tool-sample-app/env/.env.testtool
@@ -1,3 +1,2 @@
 TEAMSFX_ENV=testtool
-TEAMSAPPTESTER_PORT=56150
-TEAMSFX_NOTIFICATION_STORE_FILENAME=".notification.testtool.json"
\ No newline at end of file
+TEAMSAPPTESTER_PORT=56150
\ No newline at end of file
diff --git a/test-tool-sample-app/package.json b/test-tool-sample-app/package.json
index 4dc08c89..b253b1ae 100644
--- a/test-tool-sample-app/package.json
+++ b/test-tool-sample-app/package.json
@@ -24,7 +24,7 @@
     },
     "dependencies": {
         "@microsoft/adaptivecards-tools": "^1.0.0",
-        "@microsoft/teamsfx": "^2.2.0",
+        "@microsoft/teamsfx": "^2.3.1-beta",
         "botbuilder": "^4.18.0",
         "restify": "^10"
     },
diff --git a/test-tool-sample-app/teamsapp.testtool.yml b/test-tool-sample-app/teamsapp.testtool.yml
index 0f5291d1..76f45527 100644
--- a/test-tool-sample-app/teamsapp.testtool.yml
+++ b/test-tool-sample-app/teamsapp.testtool.yml
@@ -18,11 +18,3 @@ deploy:
   - uses: cli/runNpmCommand
     with:
       args: install --no-audit
-
-  # Generate runtime environment variables
-  - uses: file/createOrUpdateEnvironmentFile
-    with:
-      target: ./.localConfigs.testTool
-      envs:
-        TEAMSAPPTESTER_PORT: ${{TEAMSAPPTESTER_PORT}}
-        TEAMSFX_NOTIFICATION_STORE_FILENAME: ${{TEAMSFX_NOTIFICATION_STORE_FILENAME}}
\ No newline at end of file