From db21166ceabdf87e40151e5698f3c1bf7983f3d9 Mon Sep 17 00:00:00 2001
From: Chuxi Wang <wangchuxi94@gmail.com>
Date: Tue, 2 Aug 2022 18:31:26 -0700
Subject: [PATCH 01/17] add model viz to pipeline

---
 .../backend/dashboard_aws_helper.py           | 15 +++++++
 .../dashboard_app/backend/dashboard_server.py | 15 ++++++-
 .../pipeline/detail/asset/modelCard.js        | 14 +++---
 .../component/pipeline/metaInfo/modelViz.js   | 44 +++++++++++++++++++
 .../src/component/pipeline/panel.js           |  4 ++
 .../src/constants/pipelineConstants.js        |  5 +++
 6 files changed, 88 insertions(+), 9 deletions(-)
 create mode 100644 droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js

diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
index c1e1a196c4..43b6fb1fc9 100644
--- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
+++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
@@ -230,3 +230,18 @@ def get_best_model_name(batch_id: int):
     # read best model log file
     epoch_ls, text_span_ls = read_model_log_to_list(best_model_log_fname)
     return {"epoch": epoch_ls, "text_span": text_span_ls}, None
+
+
+def get_best_models_by_pipeline(pipeline: str):
+    """
+    Get best models batch_id list for a pipeline.
+    The pipeline can be nlu, tao or vision
+    """
+    local_fname = _download_file(f"{pipeline}_model_viz_batch_list.txt")
+    if local_fname is None:
+        return f"Cannot find best model list for {pipeline}", 404
+    
+    model_batch_id_list = _read_file(local_fname).split("\n")
+    
+    return model_batch_id_list, None
+
diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
index 0fc95d6d8c..a41825ce61 100644
--- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
+++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
@@ -10,6 +10,7 @@
 import json
 from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import (
     get_best_model_loss_acc_by_id,
+    get_best_models_by_pipeline,
     get_dataset_by_name,
     get_dataset_indices_by_id,
     get_dataset_version_list_by_pipeline,
@@ -50,6 +51,7 @@ class DASHBOARD_EVENT(Enum):
     GET_MODEL_KEYS = "get_model_keys_by_id"
     GET_MODEL_VALUE = "get_model_value_by_id_n_key"
     GET_BEST_MODEL_LOSS_ACC = "get_best_model_loss_acc_by_id"
+    GET_BEST_MODELS = "get_best_model_bids_by_pipeline"
 
 
 # constants for model related apis
@@ -247,7 +249,18 @@ def get_best_model_loss_acc(batch_id: int):
     if error_code:
         emit(DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value, error_code)
     else:
-        emit(DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value, loss_acc_dict)
+        emit(DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value, json.dumps([loss_acc_dict, int(batch_id)]))
+
+@socketio.on(DASHBOARD_EVENT.GET_BEST_MODELS.value)
+def get_best_model_batches(pipeline: str):
+    print(
+        f"Request received: {DASHBOARD_EVENT.GET_BEST_MODELS.value}, pipeline = {pipeline}"
+    )
+    model_dict, error_code = get_best_models_by_pipeline(pipeline)
+    if error_code:
+        emit(DASHBOARD_EVENT.GET_BEST_MODELS.value, error_code)
+    else:
+        emit(DASHBOARD_EVENT.GET_BEST_MODELS.value, model_dict)
 
 
 if __name__ == "__main__":
diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
index 93bb6e571d..1127605a9c 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
@@ -13,9 +13,8 @@ import ModelAtrributeModal from "./modelAttributeDetailModal";
 import { LineChart, Line, XAxis, YAxis, CartesianGrid, Legend, Tooltip as ChartTooltip } from 'recharts';
 
 const { Meta } = Card;
-const LOSS_ACC_TYPES = [{ "label": "Epoch", "value": "epoch" }, { "label": "Text Span", "value": "text_span" }]
 
-const ModelLossAccGraph = (props) => {
+export const ModelLossAccGraph = (props) => {
     const data = props.data.map((o, idx) => ({ Loss: o.loss, Accuracy: o.acc, Epoch: idx }));
 
     return <div style={{ width: "100%", height: "100%" }}>
@@ -41,8 +40,11 @@ const ModelLossAccGraph = (props) => {
     </div>
 }
 
-const ViewLossAccCard = (props) => {
+export const ViewLossAccCard = (props) => {
     const lossAccData = props.data;
+
+    const LOSS_ACC_TYPES = [{ "label": "Epoch", "value": "epoch" }, { "label": "Text Span", "value": "text_span" }]
+
     const [activeTabKey, setActiveTabKey] = useState(LOSS_ACC_TYPES[0]["value"]);
 
     return <Card
@@ -86,7 +88,7 @@ const ModelCard = (props) => {
     const handleReceivedLossAcc = useCallback((data) => {
         setLoadingLossAcc(false);
         if (data !== 404) {
-            setLossAccData(data);
+            setLossAccData(data[0]);
         }
     });
 
@@ -130,10 +132,6 @@ const ModelCard = (props) => {
         setAttrModalOpen(true);
     }
 
-    const handleViewModelLossAndAcc = (lossAccType) => {
-        alert(lossAccData[lossAccType].map((o) => (`loss: ${o.loss},  acc: ${o.acc}`)));
-    }
-
     return (
         <div style={{ width: '70%' }}>
             <Card title="Model" loading={loadingKeys || loadingArgs || loadingLossAcc}>
diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
new file mode 100644
index 0000000000..ad15c42fcc
--- /dev/null
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
@@ -0,0 +1,44 @@
+import React, { useContext, useEffect, useState } from "react";
+import { useLocation } from "react-router-dom";
+import { SocketContext } from "../../../context/socket";
+
+const PipelineModelVizContent = (props) => {
+    const location = useLocation();
+    const socket = useContext(SocketContext);
+    const [modelBids, setModelBids] = useState(null);
+    const [currentIdx, setCurrentIdx] = useState(0);
+    const [modelDict, setModelDict] = useState({});
+    
+    const handleReceivedModelBids = (data) => {
+        console.log(data)
+        setModelBids(data.map((bid) => parseInt(bid)));
+    }
+
+    const handleReceivedModelLossAcc = (data) => {
+        console.log(data);
+        if (modelBids && modelBids !== 404 && currentIdx >= 0 && currentIdx + 1 < modelBids.length) {
+            setCurrentIdx(currentIdx + 1);
+        }
+    }
+
+    useEffect(() => {
+        socket.on("get_best_model_bids_by_pipeline", (data) => handleReceivedModelBids(data));
+        socket.on("get_best_model_loss_acc_by_id", (data) => handleReceivedModelLossAcc(data));
+    }, [socket, handleReceivedModelBids, handleReceivedModelLossAcc]);
+
+    useEffect(() => {
+        socket.emit("get_best_model_bids_by_pipeline", location.pathname.substring(1));
+    }, []);
+
+    useEffect(() => {
+        if (modelBids && modelBids !== 404 && currentIdx >= 0) {
+            socket.emit("get_best_model_loss_acc_by_id", modelBids[currentIdx]);
+        }
+    }, [modelBids, currentIdx]);
+
+    return <div>
+        Model
+    </div>
+}
+
+export default PipelineModelVizContent;
\ No newline at end of file
diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/panel.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/panel.js
index d944740dcb..16b606cb13 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/panel.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/panel.js
@@ -13,6 +13,7 @@ import React, { useEffect, useState } from 'react';
 import { Link, Outlet, useLocation, useParams } from 'react-router-dom';
 import { TAB_ITEMS } from '../../constants/pipelineConstants';
 import InfoBlock from './metaInfo/infoBlock';
+import PipelineModelVizContent from './metaInfo/modelViz';
 import RunList from './metaInfo/runList';
 
 const menuItems = Object.values(TAB_ITEMS);
@@ -107,6 +108,9 @@ const PipelinePanel = (props) => {
                                                 // render job list if key is jobs, otherwise render info block
                                                 item.key === TAB_ITEMS.RUNS.key ?
                                                     <RunList pipelineType={pipelineType} /> :
+                                                    item.key === TAB_ITEMS.MODEL.key ? 
+                                                    <PipelineModelVizContent />
+                                                    : 
                                                     <InfoBlock infoType={item.key} pipelineType={pipelineType} />
                                             }
                                         </TabPane>
diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/constants/pipelineConstants.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/constants/pipelineConstants.js
index 604df07ee8..31f03d58d4 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/constants/pipelineConstants.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/constants/pipelineConstants.js
@@ -23,5 +23,10 @@ export const TAB_ITEMS = {
     {
         label: 'View Runs',
         key: 'runs'
+    },
+    MODEL: 
+    {
+        label: 'View Model',
+        key: 'model'
     }
 }
\ No newline at end of file

From a31d07fbb3c3dd484940766e2b0e1aad28a09f23 Mon Sep 17 00:00:00 2001
From: bot <bot@fb.com>
Date: Wed, 3 Aug 2022 01:32:14 +0000
Subject: [PATCH 02/17] Automatic style fix for droidlet

---
 .../hitl/dashboard_app/backend/dashboard_aws_helper.py |  5 ++---
 .../hitl/dashboard_app/backend/dashboard_server.py     | 10 ++++++----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
index 43b6fb1fc9..e5efa9b8cc 100644
--- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
+++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
@@ -240,8 +240,7 @@ def get_best_models_by_pipeline(pipeline: str):
     local_fname = _download_file(f"{pipeline}_model_viz_batch_list.txt")
     if local_fname is None:
         return f"Cannot find best model list for {pipeline}", 404
-    
+
     model_batch_id_list = _read_file(local_fname).split("\n")
-    
-    return model_batch_id_list, None
 
+    return model_batch_id_list, None
diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
index a41825ce61..58b5f0b5dd 100644
--- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
+++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
@@ -249,13 +249,15 @@ def get_best_model_loss_acc(batch_id: int):
     if error_code:
         emit(DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value, error_code)
     else:
-        emit(DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value, json.dumps([loss_acc_dict, int(batch_id)]))
+        emit(
+            DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value,
+            json.dumps([loss_acc_dict, int(batch_id)]),
+        )
+
 
 @socketio.on(DASHBOARD_EVENT.GET_BEST_MODELS.value)
 def get_best_model_batches(pipeline: str):
-    print(
-        f"Request received: {DASHBOARD_EVENT.GET_BEST_MODELS.value}, pipeline = {pipeline}"
-    )
+    print(f"Request received: {DASHBOARD_EVENT.GET_BEST_MODELS.value}, pipeline = {pipeline}")
     model_dict, error_code = get_best_models_by_pipeline(pipeline)
     if error_code:
         emit(DASHBOARD_EVENT.GET_BEST_MODELS.value, error_code)

From 5a9395cc1b3b1df32a3826dfa4f02e8fdf923cd3 Mon Sep 17 00:00:00 2001
From: Chuxi Wang <wangchuxi94@gmail.com>
Date: Wed, 3 Aug 2022 11:04:59 -0700
Subject: [PATCH 03/17] fixed issue due to nan

---
 .../dashboard_app/backend/dashboard_server.py    |  2 +-
 .../src/component/pipeline/metaInfo/modelViz.js  | 16 ++++++++++++++--
 droidlet/tools/hitl/utils/read_model_log.py      |  6 ++++--
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
index a41825ce61..914dd68d07 100644
--- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
+++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
@@ -249,7 +249,7 @@ def get_best_model_loss_acc(batch_id: int):
     if error_code:
         emit(DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value, error_code)
     else:
-        emit(DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value, json.dumps([loss_acc_dict, int(batch_id)]))
+        emit(DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value, [loss_acc_dict, int(batch_id)])
 
 @socketio.on(DASHBOARD_EVENT.GET_BEST_MODELS.value)
 def get_best_model_batches(pipeline: str):
diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
index ad15c42fcc..8bf6ed1e95 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
@@ -8,6 +8,7 @@ const PipelineModelVizContent = (props) => {
     const [modelBids, setModelBids] = useState(null);
     const [currentIdx, setCurrentIdx] = useState(0);
     const [modelDict, setModelDict] = useState({});
+    const [loadingPrecent, setPercent] = useState(0);
     
     const handleReceivedModelBids = (data) => {
         console.log(data)
@@ -15,10 +16,20 @@ const PipelineModelVizContent = (props) => {
     }
 
     const handleReceivedModelLossAcc = (data) => {
+        // data = JSON.parse(data);
         console.log(data);
-        if (modelBids && modelBids !== 404 && currentIdx >= 0 && currentIdx + 1 < modelBids.length) {
-            setCurrentIdx(currentIdx + 1);
+
+        if (data !== 404) {
+            modelDict[data[1]] = data[0];
+            console.log(modelDict)
+            setPercent(Object.keys(modelDict).length);
+            console.log(Object.keys(modelDict).length);
+
+            if (modelBids && modelBids !== 404 && currentIdx >= 0 && currentIdx + 1 < modelBids.length) {
+                setCurrentIdx(currentIdx + 1);
+            }
         }
+        
     }
 
     useEffect(() => {
@@ -38,6 +49,7 @@ const PipelineModelVizContent = (props) => {
 
     return <div>
         Model
+        {loadingPrecent}
     </div>
 }
 
diff --git a/droidlet/tools/hitl/utils/read_model_log.py b/droidlet/tools/hitl/utils/read_model_log.py
index 9e3bb952d3..d2eada4e12 100644
--- a/droidlet/tools/hitl/utils/read_model_log.py
+++ b/droidlet/tools/hitl/utils/read_model_log.py
@@ -3,7 +3,7 @@
 
 Utils for processing model log.
 """
-
+import math
 
 def read_model_log_to_list(fname: str):
     """
@@ -23,8 +23,10 @@ def get_loss_acc(line: str):
         for i in range(0, len(words)):
             if i > 0 and words[i - 1] == "Loss:":
                 loss = float(words[i])
+                loss = None if math.isnan(loss) else loss
             elif i > 0 and words[i - 1] == "Accuracy:":
-                acc = float(words[i])
+                acc = float(words[i]) 
+                acc = None if acc is math.isnan(acc) else acc
         return loss, acc
 
     epoch_loss_acc_list = []

From 7029c2305b1db99b188c64ecbeb4fef5860a952c Mon Sep 17 00:00:00 2001
From: Chuxi Wang <wangchuxi94@gmail.com>
Date: Wed, 3 Aug 2022 14:23:56 -0700
Subject: [PATCH 04/17] add graph display of loss and acc change

---
 .../backend/dashboard_aws_helper.py           |  1 +
 .../component/pipeline/metaInfo/modelViz.js   | 54 ++++++++++++++-----
 2 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
index 43b6fb1fc9..4fbc7dd07b 100644
--- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
+++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
@@ -229,6 +229,7 @@ def get_best_model_name(batch_id: int):
 
     # read best model log file
     epoch_ls, text_span_ls = read_model_log_to_list(best_model_log_fname)
+    print(f"epoch length: {len(epoch_ls)}, text_span length: {len(text_span_ls)}")
     return {"epoch": epoch_ls, "text_span": text_span_ls}, None
 
 
diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
index 8bf6ed1e95..837c095821 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
@@ -1,6 +1,9 @@
+import { SyncOutlined } from "@ant-design/icons";
+import { Progress, Typography } from "antd";
 import React, { useContext, useEffect, useState } from "react";
 import { useLocation } from "react-router-dom";
 import { SocketContext } from "../../../context/socket";
+import { ViewLossAccCard } from "../detail/asset/modelCard";
 
 const PipelineModelVizContent = (props) => {
     const location = useLocation();
@@ -8,28 +11,37 @@ const PipelineModelVizContent = (props) => {
     const [modelBids, setModelBids] = useState(null);
     const [currentIdx, setCurrentIdx] = useState(0);
     const [modelDict, setModelDict] = useState({});
-    const [loadingPrecent, setPercent] = useState(0);
-    
+    const [modelCombined, setModelCombined] = useState(null);
+
     const handleReceivedModelBids = (data) => {
-        console.log(data)
         setModelBids(data.map((bid) => parseInt(bid)));
     }
 
     const handleReceivedModelLossAcc = (data) => {
-        // data = JSON.parse(data);
-        console.log(data);
-
         if (data !== 404) {
             modelDict[data[1]] = data[0];
-            console.log(modelDict)
-            setPercent(Object.keys(modelDict).length);
-            console.log(Object.keys(modelDict).length);
 
-            if (modelBids && modelBids !== 404 && currentIdx >= 0 && currentIdx + 1 < modelBids.length) {
-                setCurrentIdx(currentIdx + 1);
+            if (modelBids && modelBids !== 404 && Object.keys(modelDict).length < modelBids.length) {
+                setCurrentIdx(Object.keys(modelDict).length);
+            } else if (modelBids && modelBids !== 404 && Object.keys(modelDict).length === modelBids.length) {
+                // finished loading, combine based on bid sequence
+                const modelArr = modelBids.map((bid) => (modelDict[bid]));
+                let comb = {}
+
+                for (let i = 0; i < modelArr.length; i++) {
+                    const o = modelArr[i];
+                    for (let j = 0; j < Object.keys(o).length; j++) {
+                        const key = Object.keys(o)[j];
+                        if (!(key in comb)) {
+                            comb[key] = o[key];
+                        } else {
+                            comb[key] = [...comb[key], ...o[key]]
+                        }
+                    }
+                }
+                setModelCombined(comb);
             }
         }
-        
     }
 
     useEffect(() => {
@@ -48,8 +60,22 @@ const PipelineModelVizContent = (props) => {
     }, [modelBids, currentIdx]);
 
     return <div>
-        Model
-        {loadingPrecent}
+        <Typography.Title level={4}>{"View Model Accuracy & Loss"}</Typography.Title>
+        {
+            // show loading model progress
+            modelBids && modelBids !== 404 && (currentIdx + 1) !== modelBids.length &&
+            <div style={{ padding: "0 24px 0 24px" }}>
+                <span><SyncOutlined spin /> Loading Model Data... </span>
+                <Progress percent={Math.trunc((currentIdx + 1) / modelBids.length * 100)} />
+            </div>
+        }
+        {
+            // show graph when loading is done
+            modelBids && modelBids !== 404 && (currentIdx + 1) === modelBids.length && modelCombined &&
+            <div style={{ width: '1200px', height: '500px' }}>
+                <ViewLossAccCard data={modelCombined} />
+            </div>
+        }
     </div>
 }
 

From 378cb24f66ddeba38fd1fd8d8a4976e7a14c92b3 Mon Sep 17 00:00:00 2001
From: bot <bot@fb.com>
Date: Wed, 3 Aug 2022 21:25:43 +0000
Subject: [PATCH 05/17] Automatic style fix for droidlet

---
 droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py | 1 +
 droidlet/tools/hitl/utils/read_model_log.py                   | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
index b5298c0dcd..bdb3c2f20b 100644
--- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
+++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
@@ -251,6 +251,7 @@ def get_best_model_loss_acc(batch_id: int):
     else:
         emit(DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value, [loss_acc_dict, int(batch_id)])
 
+
 @socketio.on(DASHBOARD_EVENT.GET_BEST_MODELS.value)
 def get_best_model_batches(pipeline: str):
     print(f"Request received: {DASHBOARD_EVENT.GET_BEST_MODELS.value}, pipeline = {pipeline}")
diff --git a/droidlet/tools/hitl/utils/read_model_log.py b/droidlet/tools/hitl/utils/read_model_log.py
index d2eada4e12..3b2b3ccbb6 100644
--- a/droidlet/tools/hitl/utils/read_model_log.py
+++ b/droidlet/tools/hitl/utils/read_model_log.py
@@ -5,6 +5,7 @@
 """
 import math
 
+
 def read_model_log_to_list(fname: str):
     """
     Read model log and export to a dictionary including:
@@ -25,7 +26,7 @@ def get_loss_acc(line: str):
                 loss = float(words[i])
                 loss = None if math.isnan(loss) else loss
             elif i > 0 and words[i - 1] == "Accuracy:":
-                acc = float(words[i]) 
+                acc = float(words[i])
                 acc = None if acc is math.isnan(acc) else acc
         return loss, acc
 

From 2e7ca61d0672b276e2e0ca6cfcada50a17edae61 Mon Sep 17 00:00:00 2001
From: Chuxi Wang <wangchuxi94@gmail.com>
Date: Thu, 4 Aug 2022 14:04:49 -0700
Subject: [PATCH 06/17] add documentation

---
 .../backend/dashboard_aws_helper.py           |  1 -
 .../dashboard_app/backend/dashboard_server.py | 15 ++++--
 .../pipeline/detail/asset/modelCard.js        | 11 ++--
 .../component/pipeline/metaInfo/modelViz.js   | 53 +++++++++++++------
 4 files changed, 56 insertions(+), 24 deletions(-)

diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
index 0094d70bbe..e7d580494d 100644
--- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
+++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
@@ -202,7 +202,6 @@ def get_best_model_loss_acc_by_id(batch_id: int):
         - a dict including epoch & text span loss and accuracy, and no error code if can find the best model log
         - an error message with error code 404 if cannot find the best model log
     """
-
     def get_best_model_name(batch_id: int):
         # download and read best_model.txt file
         best_model_fname = ""
diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
index bdb3c2f20b..139aa8ba5d 100644
--- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
+++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
@@ -237,9 +237,11 @@ def get_best_model_loss_acc(batch_id: int):
     - input:
         - batch id of a specific run
     - output:
-        - a dictionary containing:
-            - epoch loss and accuracy
-            - text_span loss and accuracy
+        - a list contains the following when a best model log file can be found:
+            - a dictionary containing:
+                - epoch loss and accuracy
+                - text_span loss and accuracy
+            - batch id 
         - or an error code indicating the best model log cannot be find
     """
     print(
@@ -254,6 +256,13 @@ def get_best_model_loss_acc(batch_id: int):
 
 @socketio.on(DASHBOARD_EVENT.GET_BEST_MODELS.value)
 def get_best_model_batches(pipeline: str):
+    """
+    get best models' batch ids for a specific pipeline
+    - input:
+        - pipeline name (can be nlu, tao or vision)
+    - output:
+        - a list of the batch ids of the best models
+    """
     print(f"Request received: {DASHBOARD_EVENT.GET_BEST_MODELS.value}, pipeline = {pipeline}")
     model_dict, error_code = get_best_models_by_pipeline(pipeline)
     if error_code:
diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
index 1127605a9c..5fca58d4bb 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
@@ -15,12 +15,14 @@ import { LineChart, Line, XAxis, YAxis, CartesianGrid, Legend, Tooltip as ChartT
 const { Meta } = Card;
 
 export const ModelLossAccGraph = (props) => {
+    const width = props.width;
+    const height = props.height;
     const data = props.data.map((o, idx) => ({ Loss: o.loss, Accuracy: o.acc, Epoch: idx }));
 
     return <div style={{ width: "100%", height: "100%" }}>
         <LineChart
-            width={750}
-            height={400}
+            width={width}
+            height={height}
             data={data}
             margin={{
                 top: 5,
@@ -41,6 +43,8 @@ export const ModelLossAccGraph = (props) => {
 }
 
 export const ViewLossAccCard = (props) => {
+    const width = props.width ? props.width : 750;
+    const height = props.height ? props.height : 400;
     const lossAccData = props.data;
 
     const LOSS_ACC_TYPES = [{ "label": "Epoch", "value": "epoch" }, { "label": "Text Span", "value": "text_span" }]
@@ -52,13 +56,12 @@ export const ViewLossAccCard = (props) => {
         activeTabKey={activeTabKey}
         onTabChange={(key) => setActiveTabKey(key)}
     >
-        <ModelLossAccGraph data={lossAccData[activeTabKey]} />
+        <ModelLossAccGraph data={lossAccData[activeTabKey]} height = {height} width = {width} />
     </Card>
 }
 
 const ModelCard = (props) => {
     const batchId = props.batchId;
-    const pipelineType = props.pipelineType;
     const [modelArgs, setModelArgs] = useState(null);
     const [modelKeys, setModelKeys] = useState(null);
     const [loadingArgs, setLoadingArgs] = useState(true);
diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
index 837c095821..b2db208a01 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
@@ -1,3 +1,14 @@
+/*
+Copyright (c) Facebook, Inc. and its affiliates.
+
+The content for model visualization for a pipeline.
+
+The **pipeline** can be nlu/tao or vision; the pipeline information was reterived from path parameter so no props are required.
+
+Usage:
+<PipelineModelVizContent />
+*/
+
 import { SyncOutlined } from "@ant-design/icons";
 import { Progress, Typography } from "antd";
 import React, { useContext, useEffect, useState } from "react";
@@ -10,7 +21,7 @@ const PipelineModelVizContent = (props) => {
     const socket = useContext(SocketContext);
     const [modelBids, setModelBids] = useState(null);
     const [currentIdx, setCurrentIdx] = useState(0);
-    const [modelDict, setModelDict] = useState({});
+    const [modelDict, ] = useState({});
     const [modelCombined, setModelCombined] = useState(null);
 
     const handleReceivedModelBids = (data) => {
@@ -61,21 +72,31 @@ const PipelineModelVizContent = (props) => {
 
     return <div>
         <Typography.Title level={4}>{"View Model Accuracy & Loss"}</Typography.Title>
-        {
-            // show loading model progress
-            modelBids && modelBids !== 404 && (currentIdx + 1) !== modelBids.length &&
-            <div style={{ padding: "0 24px 0 24px" }}>
-                <span><SyncOutlined spin /> Loading Model Data... </span>
-                <Progress percent={Math.trunc((currentIdx + 1) / modelBids.length * 100)} />
-            </div>
-        }
-        {
-            // show graph when loading is done
-            modelBids && modelBids !== 404 && (currentIdx + 1) === modelBids.length && modelCombined &&
-            <div style={{ width: '1200px', height: '500px' }}>
-                <ViewLossAccCard data={modelCombined} />
-            </div>
-        }
+        <div style={{ padding: "0 24px 0 24px" }}>
+            {
+                // show progress bar after getting model batch ids
+                modelBids && modelBids !== 404 && (Object.keys(modelDict).length) !== modelBids.length &&
+                <div> <span><SyncOutlined spin /> Loading Model Data... </span>
+                    <Progress percent={Math.trunc((Object.keys(modelDict).length) / modelBids.length * 100)} />
+                </div>
+            }
+
+            {
+                // show graph when loading is done
+                modelBids && modelBids !== 404 && (Object.keys(modelDict).length) === modelBids.length && modelCombined &&
+                <div>
+                    <ViewLossAccCard data={modelCombined} width={1500}/>
+                </div>
+            } 
+            {
+                // show no data when not having any batch ids
+                modelBids && modelBids === 404 && 
+                <div>
+                    Sorry, no model data is avaible yet. 
+                </div>
+            }
+        </div>
+
     </div>
 }
 

From b686c0761c12ee5baf7db16d17a14adff9b5768d Mon Sep 17 00:00:00 2001
From: Chuxi Wang <wangchuxi94@gmail.com>
Date: Thu, 4 Aug 2022 16:43:30 -0700
Subject: [PATCH 07/17] update graph to show loss and accuracy of the
 validation and training set

---
 .../backend/dashboard_aws_helper.py           |  8 ++--
 .../dashboard_app/backend/dashboard_server.py |  4 +-
 .../pipeline/detail/asset/modelCard.js        | 36 ++++++++++------
 .../component/pipeline/metaInfo/modelViz.js   | 14 ++++++-
 droidlet/tools/hitl/utils/read_model_log.py   | 42 ++++++++++---------
 5 files changed, 64 insertions(+), 40 deletions(-)

diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
index e7d580494d..709151b2b0 100644
--- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
+++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
@@ -199,7 +199,7 @@ def get_best_model_loss_acc_by_id(batch_id: int):
     """
     Get best model loss and accuracy from model log file,
     return:
-        - a dict including epoch & text span loss and accuracy, and no error code if can find the best model log
+        - a dict including loss and accuracy for training and valiation, and no error code if can find the best model log
         - an error message with error code 404 if cannot find the best model log
     """
     def get_best_model_name(batch_id: int):
@@ -227,9 +227,9 @@ def get_best_model_name(batch_id: int):
         return f"Cannot find best model log file with batch_id = {batch_id}", 404
 
     # read best model log file
-    epoch_ls, text_span_ls = read_model_log_to_list(best_model_log_fname)
-    print(f"epoch length: {len(epoch_ls)}, text_span length: {len(text_span_ls)}")
-    return {"epoch": epoch_ls, "text_span": text_span_ls}, None
+    loss_ls, acc_ls = read_model_log_to_list(best_model_log_fname)
+    print(f"loss length: {len(loss_ls)}, accuracy length: {len(acc_ls)}")
+    return {"loss": loss_ls, "acc": acc_ls}, None
 
 
 def get_best_models_by_pipeline(pipeline: str):
diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
index 139aa8ba5d..5a4c485c89 100644
--- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
+++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
@@ -239,8 +239,8 @@ def get_best_model_loss_acc(batch_id: int):
     - output:
         - a list contains the following when a best model log file can be found:
             - a dictionary containing:
-                - epoch loss and accuracy
-                - text_span loss and accuracy
+                - key: loss, value: loss list for training and validation dataset
+                - key: acc, value: accuracy list for training and validation dataset
             - batch id 
         - or an error code indicating the best model log cannot be find
     """
diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
index 5fca58d4bb..0a263e5ecb 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
@@ -10,16 +10,26 @@ import { Button, Card, Descriptions, Divider, Tooltip, Typography } from "antd";
 import React, { useCallback, useContext, useEffect, useState } from "react";
 import { SocketContext } from "../../../../context/socket";
 import ModelAtrributeModal from "./modelAttributeDetailModal";
-import { LineChart, Line, XAxis, YAxis, CartesianGrid, Legend, Tooltip as ChartTooltip } from 'recharts';
+import { LineChart, Line, XAxis, YAxis, CartesianGrid, Legend, Tooltip as ChartTooltip, ResponsiveContainer } from "recharts";
 
 const { Meta } = Card;
 
 export const ModelLossAccGraph = (props) => {
     const width = props.width;
     const height = props.height;
-    const data = props.data.map((o, idx) => ({ Loss: o.loss, Accuracy: o.acc, Epoch: idx }));
+    const yAxisName = props.yAxisName;
 
-    return <div style={{ width: "100%", height: "100%" }}>
+    const getScaled = (yName, value) => {
+        // if is loss, get log2 scaled
+        if ((yName) === "Loss") {
+            return value ? Math.log2(value): 0;
+        } else {
+            return value;
+        }
+    }
+    const data = props.data.map((o, idx) => ({ Training: getScaled(yAxisName, o.training), Validation:  getScaled(yAxisName, o.validation), Epoch: idx }));
+
+    return <ResponsiveContainer height={"70%"} aspect={width / height}>
         <LineChart
             width={width}
             height={height}
@@ -33,13 +43,13 @@ export const ModelLossAccGraph = (props) => {
         >
             <CartesianGrid strokeDasharray="3 3" />
             <XAxis type="number" dataKey="Epoch" label={{offset: 0, value: "Epoch", position: "insideBottom" }}/>
-            <YAxis />
+            <YAxis label={{offset: 0, value: (yAxisName === "Loss" ? yAxisName + "(Log2 Scaled)": yAxisName), position: "insideLeft", angle: -90}}/>
             <Legend />
             <ChartTooltip />
-            <Line type="monotone" dataKey="Loss" stroke="#ad2102" />
-            <Line type="monotone" dataKey="Accuracy" stroke="#1890ff" activeDot={{ r: 8 }} dot={{strokeWidth: 2}} strokeWidth={2} />
+            <Line type="monotone" dataKey="Training" stroke="#ad2102" />
+            <Line type="monotone" dataKey="Validation" stroke="#1890ff" activeDot={{ r: 8 }} dot={{strokeWidth: 2}} strokeWidth={2} />
         </LineChart>
-    </div>
+    </ResponsiveContainer>
 }
 
 export const ViewLossAccCard = (props) => {
@@ -47,16 +57,16 @@ export const ViewLossAccCard = (props) => {
     const height = props.height ? props.height : 400;
     const lossAccData = props.data;
 
-    const LOSS_ACC_TYPES = [{ "label": "Epoch", "value": "epoch" }, { "label": "Text Span", "value": "text_span" }]
+    const LOSS_ACC_TYPES = [{ "label": "Accuracy", "value": "acc" }, { "label": "Loss", "value": "loss" }]
 
     const [activeTabKey, setActiveTabKey] = useState(LOSS_ACC_TYPES[0]["value"]);
 
     return <Card
         tabList={LOSS_ACC_TYPES.map((o) => ({ tab: o["label"], key: o["value"] }))}
         activeTabKey={activeTabKey}
-        onTabChange={(key) => setActiveTabKey(key)}
+        onTabChange={(key) => {setActiveTabKey(key)}}
     >
-        <ModelLossAccGraph data={lossAccData[activeTabKey]} height = {height} width = {width} />
+        <ModelLossAccGraph data={lossAccData[activeTabKey]} height = {height} width = {width} yAxisName={LOSS_ACC_TYPES.find((o) => (o.value === activeTabKey))["label"]} />
     </Card>
 }
 
@@ -136,7 +146,7 @@ const ModelCard = (props) => {
     }
 
     return (
-        <div style={{ width: '70%' }}>
+        <div style={{ width: "70%" }}>
             <Card title="Model" loading={loadingKeys || loadingArgs || loadingLossAcc}>
                 <Meta />
                 {
@@ -166,7 +176,7 @@ const ModelCard = (props) => {
                                     lossAccData &&
                                     <>
                                         <Divider />
-                                        <Typography.Title level={5} style={{ paddingBottom: '18px' }}>Model Loss And Accuracy</Typography.Title>
+                                        <Typography.Title level={5} style={{ paddingBottom: "18px" }}>Model Loss And Accuracy</Typography.Title>
                                         <ViewLossAccCard data={lossAccData} />
                                     </>
                                 }
@@ -175,7 +185,7 @@ const ModelCard = (props) => {
                             <div>NA</div>
                     )
                 }
-                {/* modal showing a specific model attribute's field (anything other than args) */}
+                {/* modal showing a specific model attribute"s field (anything other than args) */}
                 {currentModelKey
                     &&
                     <ModelAtrributeModal
diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
index b2db208a01..6adb9b8d98 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
@@ -10,7 +10,7 @@ Usage:
 */
 
 import { SyncOutlined } from "@ant-design/icons";
-import { Progress, Typography } from "antd";
+import { Progress, Tag, Typography } from "antd";
 import React, { useContext, useEffect, useState } from "react";
 import { useLocation } from "react-router-dom";
 import { SocketContext } from "../../../context/socket";
@@ -72,11 +72,21 @@ const PipelineModelVizContent = (props) => {
 
     return <div>
         <Typography.Title level={4}>{"View Model Accuracy & Loss"}</Typography.Title>
+        {
+            modelBids && modelBids !== 404 
+            && 
+            <div
+                style={{padding: "0 0 12px 0"}}
+            >
+                Showing {modelBids.map((bid) => <Tag>{bid}</Tag>)} 
+            </div>
+        }
         <div style={{ padding: "0 24px 0 24px" }}>
             {
                 // show progress bar after getting model batch ids
                 modelBids && modelBids !== 404 && (Object.keys(modelDict).length) !== modelBids.length &&
-                <div> <span><SyncOutlined spin /> Loading Model Data... </span>
+                <div> 
+                    <span><SyncOutlined spin /> Loading Model Data... </span>
                     <Progress percent={Math.trunc((Object.keys(modelDict).length) / modelBids.length * 100)} />
                 </div>
             }
diff --git a/droidlet/tools/hitl/utils/read_model_log.py b/droidlet/tools/hitl/utils/read_model_log.py
index 3b2b3ccbb6..9964476da0 100644
--- a/droidlet/tools/hitl/utils/read_model_log.py
+++ b/droidlet/tools/hitl/utils/read_model_log.py
@@ -9,35 +9,39 @@
 def read_model_log_to_list(fname: str):
     """
     Read model log and export to a dictionary including:
-        - Epoch loss and accuracy
-        - Text span loss and accuracy
+        - Loss changes over epoch increases of training / validation
+        - Accuracy changes over epoch increases of training / validation
     """
-    f = open(fname)
-    content = f.read()
-    f.close()
-
-    lines = content.split("\n")
-
     def get_loss_acc(line: str):
         words = line.split(" ")
         loss, acc = None, None
         for i in range(0, len(words)):
-            if i > 0 and words[i - 1] == "Loss:":
+            if i > 0 and (words[i - 1] == "Loss:" or words[i - 1] == "L:"):
                 loss = float(words[i])
                 loss = None if math.isnan(loss) else loss
-            elif i > 0 and words[i - 1] == "Accuracy:":
+            elif i > 0 and (words[i - 1] == "Accuracy:" or words[i - 1] == "A:"):
                 acc = float(words[i])
                 acc = None if acc is math.isnan(acc) else acc
         return loss, acc
 
-    epoch_loss_acc_list = []
-    text_span_loss_acc_list = []
-    for line in lines:
-        if "epoch" in line:
+    loss_list = []  
+    acc_list = []
+    t_key, v_key = "training", "validation"
+
+    f = open(fname)
+    for line in f:
+        if "Epoch: " in line:
+            loss_list.append({})
+            acc_list.append({})
+        elif "L: " in line:
+            # training
             loss, acc = get_loss_acc(line)
-            epoch_loss_acc_list.append({"loss": loss, "acc": acc})
-        elif " text span Loss: " in line:
+            loss_list[-1][t_key] = loss
+            acc_list[-1][t_key] = acc
+        elif "valid: " in line:
+            # validation
             loss, acc = get_loss_acc(line)
-            text_span_loss_acc_list.append({"loss": loss, "acc": acc})
-
-    return epoch_loss_acc_list, text_span_loss_acc_list
+            loss_list[-1][v_key] = loss
+            acc_list[-1][v_key] = acc
+    f.close()
+    return loss_list, acc_list

From 242806f3231c1fb4263fc545b98c296b275f9af3 Mon Sep 17 00:00:00 2001
From: bot <bot@fb.com>
Date: Thu, 4 Aug 2022 23:44:16 +0000
Subject: [PATCH 08/17] Automatic style fix for droidlet

---
 .../tools/hitl/dashboard_app/backend/dashboard_aws_helper.py   | 1 +
 droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py  | 2 +-
 droidlet/tools/hitl/utils/read_model_log.py                    | 3 ++-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
index 709151b2b0..cd88300e88 100644
--- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
+++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py
@@ -202,6 +202,7 @@ def get_best_model_loss_acc_by_id(batch_id: int):
         - a dict including loss and accuracy for training and valiation, and no error code if can find the best model log
         - an error message with error code 404 if cannot find the best model log
     """
+
     def get_best_model_name(batch_id: int):
         # download and read best_model.txt file
         best_model_fname = ""
diff --git a/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py b/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
index 5a4c485c89..60fcbcedf1 100644
--- a/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
+++ b/droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py
@@ -241,7 +241,7 @@ def get_best_model_loss_acc(batch_id: int):
             - a dictionary containing:
                 - key: loss, value: loss list for training and validation dataset
                 - key: acc, value: accuracy list for training and validation dataset
-            - batch id 
+            - batch id
         - or an error code indicating the best model log cannot be find
     """
     print(
diff --git a/droidlet/tools/hitl/utils/read_model_log.py b/droidlet/tools/hitl/utils/read_model_log.py
index 9964476da0..9dc34aaec0 100644
--- a/droidlet/tools/hitl/utils/read_model_log.py
+++ b/droidlet/tools/hitl/utils/read_model_log.py
@@ -12,6 +12,7 @@ def read_model_log_to_list(fname: str):
         - Loss changes over epoch increases of training / validation
         - Accuracy changes over epoch increases of training / validation
     """
+
     def get_loss_acc(line: str):
         words = line.split(" ")
         loss, acc = None, None
@@ -24,7 +25,7 @@ def get_loss_acc(line: str):
                 acc = None if acc is math.isnan(acc) else acc
         return loss, acc
 
-    loss_list = []  
+    loss_list = []
     acc_list = []
     t_key, v_key = "training", "validation"
 

From d373e4b682ce6b89247f026ce7673e2cdaaaf7b9 Mon Sep 17 00:00:00 2001
From: Chuxi Wang <wangchuxi94@gmail.com>
Date: Thu, 4 Aug 2022 17:17:15 -0700
Subject: [PATCH 09/17] updated readme

---
 droidlet/tools/hitl/dashboard_app/README.MD | 26 ++++++++++++++++-----
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/droidlet/tools/hitl/dashboard_app/README.MD b/droidlet/tools/hitl/dashboard_app/README.MD
index 286977ce52..72152d2eea 100644
--- a/droidlet/tools/hitl/dashboard_app/README.MD
+++ b/droidlet/tools/hitl/dashboard_app/README.MD
@@ -1,9 +1,17 @@
 # Dashboard App for HITL
-Updated July 22 by Chuxi.
+Updated Aug 4 by Chuxi.
 
 This is a Dashboard app prototype for the HITL system.
 
 ## Update Note
+- Aug 4:
+    - Update support for visualizing the model:
+        - Changed from showing loss and accuracy of validation and text_span to showing loss and accuracy seperately, and show training loss/accuracy & validation loss/accuracy in the same graph. 
+        - Demo:
+            - Best models loss & accuracy for NLU pipeline:
+                - 
+            - Best model in a run:
+                - 
 - July 22:
     - Updated frontend component to show the model line graph of loss and accuracy vs epoch.
     - Demo:
@@ -128,15 +136,21 @@ APIs are based on socket event, the following APIs are supported currently:
         - the key for the model, could be any key from the model, or "COMPLETE", indicating getting the complete model dict
     - output: the key and the value specific to the key for the model if the model exists and key is valid, otherwise error code
 - get_best_model_loss_acc_by_id
-    - get loss and accuracy from a model log for a specific run's best model, 
-    the loss and accuracy infomation is reterived by crawling the best model's log file
+    - get loss and accuracy from a model log for a specific run's best model, the loss and accuracy infomation is reterived by crawling the best model's log file
     - input:
         - batch id of a specific run
     - output:
-        - a dictionary containing:
-            - epoch loss and accuracy
-            - text_span loss and accuracy
+        - a list contains the following when a best model log file can be found:
+            - a dictionary containing:
+                - key: loss, value: loss list for training and validation dataset
+                - key: acc, value: accuracy list for training and validation dataset
+            - batch id 
         - or an error code indicating the best model log cannot be find
+- get best models' batch ids for a specific pipeline
+    - input:
+        - pipeline name (can be nlu, tao or vision)
+    - output:
+        - a list of the batch ids of the best models
  
 ## Demo
 ![backend_api_demo](https://user-images.githubusercontent.com/51009396/175696481-532cec55-5b2e-4bae-bceb-9e7d3f2aa7b7.gif)

From de08e13cbe28b1dfb01c508eab1c77f6c067995f Mon Sep 17 00:00:00 2001
From: Chuxi Wang <wangchuxi94@gmail.com>
Date: Thu, 4 Aug 2022 17:23:52 -0700
Subject: [PATCH 10/17] updated readme

---
 droidlet/tools/hitl/dashboard_app/README.MD | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/droidlet/tools/hitl/dashboard_app/README.MD b/droidlet/tools/hitl/dashboard_app/README.MD
index 72152d2eea..828e93a237 100644
--- a/droidlet/tools/hitl/dashboard_app/README.MD
+++ b/droidlet/tools/hitl/dashboard_app/README.MD
@@ -7,10 +7,11 @@ This is a Dashboard app prototype for the HITL system.
 - Aug 4:
     - Update support for visualizing the model:
         - Changed from showing loss and accuracy of validation and text_span to showing loss and accuracy seperately, and show training loss/accuracy & validation loss/accuracy in the same graph. 
+        - Updated the graph visualization container to be responsive to browser window changes. 
         - Demo:
             - Best models loss & accuracy for NLU pipeline:
                 - 
-            - Best model in a run:
+            - Best model in a run (showing loss and accuracy seperately, graph responsive to browser window changes):
                 - 
 - July 22:
     - Updated frontend component to show the model line graph of loss and accuracy vs epoch.

From 09dae84b6fce94bc922fe3a92e1de52c1f046fd8 Mon Sep 17 00:00:00 2001
From: Chuxi Wang <51009396+mialsy@users.noreply.github.com>
Date: Thu, 4 Aug 2022 17:26:59 -0700
Subject: [PATCH 11/17] Update README.MD

---
 droidlet/tools/hitl/dashboard_app/README.MD | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/droidlet/tools/hitl/dashboard_app/README.MD b/droidlet/tools/hitl/dashboard_app/README.MD
index 828e93a237..09fde91b8c 100644
--- a/droidlet/tools/hitl/dashboard_app/README.MD
+++ b/droidlet/tools/hitl/dashboard_app/README.MD
@@ -9,10 +9,10 @@ This is a Dashboard app prototype for the HITL system.
         - Changed from showing loss and accuracy of validation and text_span to showing loss and accuracy seperately, and show training loss/accuracy & validation loss/accuracy in the same graph. 
         - Updated the graph visualization container to be responsive to browser window changes. 
         - Demo:
-            - Best models loss & accuracy for NLU pipeline:
-                - 
+            - Best models loss & accuracy for a pipeline:
+                - ![demo_best_models_for_pipeline](https://user-images.githubusercontent.com/51009396/182976894-a1eb380b-06fd-491b-8e4f-29abd7c2024d.gif)
             - Best model in a run (showing loss and accuracy seperately, graph responsive to browser window changes):
-                - 
+                - ![demo_best_model_for_a_run](https://user-images.githubusercontent.com/51009396/182976905-60713821-31d8-4187-889a-11b3b1d1ce82.gif)
 - July 22:
     - Updated frontend component to show the model line graph of loss and accuracy vs epoch.
     - Demo:

From ccd24550304ec9c90a1bc5474d30516be2de9f1d Mon Sep 17 00:00:00 2001
From: Chuxi Wang <wangchuxi94@gmail.com>
Date: Thu, 4 Aug 2022 17:29:08 -0700
Subject: [PATCH 12/17] fixed typo

---
 droidlet/tools/hitl/dashboard_app/README.MD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/droidlet/tools/hitl/dashboard_app/README.MD b/droidlet/tools/hitl/dashboard_app/README.MD
index 09fde91b8c..b45f958bdf 100644
--- a/droidlet/tools/hitl/dashboard_app/README.MD
+++ b/droidlet/tools/hitl/dashboard_app/README.MD
@@ -6,7 +6,7 @@ This is a Dashboard app prototype for the HITL system.
 ## Update Note
 - Aug 4:
     - Update support for visualizing the model:
-        - Changed from showing loss and accuracy of validation and text_span to showing loss and accuracy seperately, and show training loss/accuracy & validation loss/accuracy in the same graph. 
+        - Changed from showing loss and accuracy of validation and text_span to showing loss and accuracy separately, and show training loss/accuracy & validation loss/accuracy in the same graph. 
         - Updated the graph visualization container to be responsive to browser window changes. 
         - Demo:
             - Best models loss & accuracy for a pipeline:

From 8f605c8284a2320408389df9c5f6c94894fe1410 Mon Sep 17 00:00:00 2001
From: Chuxi Wang <wangchuxi94@gmail.com>
Date: Thu, 4 Aug 2022 17:30:34 -0700
Subject: [PATCH 13/17] updated readme

---
 droidlet/tools/hitl/dashboard_app/README.MD | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/droidlet/tools/hitl/dashboard_app/README.MD b/droidlet/tools/hitl/dashboard_app/README.MD
index b45f958bdf..5a965fae6b 100644
--- a/droidlet/tools/hitl/dashboard_app/README.MD
+++ b/droidlet/tools/hitl/dashboard_app/README.MD
@@ -147,7 +147,8 @@ APIs are based on socket event, the following APIs are supported currently:
                 - key: acc, value: accuracy list for training and validation dataset
             - batch id 
         - or an error code indicating the best model log cannot be find
-- get best models' batch ids for a specific pipeline
+- get_best_model_bids_by_pipeline
+    - get best models' batch ids for a specific pipeline
     - input:
         - pipeline name (can be nlu, tao or vision)
     - output:

From 6696479d0892d7d87cb451da8aa063099dd6eb36 Mon Sep 17 00:00:00 2001
From: Chuxi Wang <wangchuxi94@gmail.com>
Date: Fri, 5 Aug 2022 15:25:13 -0700
Subject: [PATCH 14/17] refactor style

---
 .../pipeline/detail/asset/modelCard.js        | 41 +++++++++++--------
 .../component/pipeline/metaInfo/modelViz.js   |  7 +++-
 2 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
index 0a263e5ecb..ab17ab9735 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
@@ -14,6 +14,18 @@ import { LineChart, Line, XAxis, YAxis, CartesianGrid, Legend, Tooltip as ChartT
 
 const { Meta } = Card;
 
+const lineChartMargin = {
+    top: 5,
+    right: 30,
+    left: 20,
+    bottom: 5,
+};
+
+const modelContainerStyle = {width: "70%"};
+const titleBottomPaddingStyle = { paddingBottom: "18px" };
+const textAlignLeftStyle = { textAlign: "left" };
+
+
 export const ModelLossAccGraph = (props) => {
     const width = props.width;
     const height = props.height;
@@ -22,32 +34,27 @@ export const ModelLossAccGraph = (props) => {
     const getScaled = (yName, value) => {
         // if is loss, get log2 scaled
         if ((yName) === "Loss") {
-            return value ? Math.log2(value): 0;
+            return value ? Math.log2(value) : 0;
         } else {
             return value;
         }
     }
-    const data = props.data.map((o, idx) => ({ Training: getScaled(yAxisName, o.training), Validation:  getScaled(yAxisName, o.validation), Epoch: idx }));
+    const data = props.data.map((o, idx) => ({ Training: getScaled(yAxisName, o.training), Validation: getScaled(yAxisName, o.validation), Epoch: idx }));
 
-    return <ResponsiveContainer height={"70%"} aspect={width / height}>
+    return <ResponsiveContainer aspect={width / height}>
         <LineChart
             width={width}
             height={height}
             data={data}
-            margin={{
-                top: 5,
-                right: 30,
-                left: 20,
-                bottom: 5,
-              }}
+            margin={lineChartMargin}
         >
             <CartesianGrid strokeDasharray="3 3" />
-            <XAxis type="number" dataKey="Epoch" label={{offset: 0, value: "Epoch", position: "insideBottom" }}/>
-            <YAxis label={{offset: 0, value: (yAxisName === "Loss" ? yAxisName + "(Log2 Scaled)": yAxisName), position: "insideLeft", angle: -90}}/>
+            <XAxis type="number" dataKey="Epoch" label={{ offset: 0, value: "Epoch", position: "insideBottom" }} />
+            <YAxis label={{ offset: 0, value: (yAxisName === "Loss" ? yAxisName + " (Log2 Scaled)" : yAxisName), position: "insideLeft", angle: -90 }} />
             <Legend />
             <ChartTooltip />
             <Line type="monotone" dataKey="Training" stroke="#ad2102" />
-            <Line type="monotone" dataKey="Validation" stroke="#1890ff" activeDot={{ r: 8 }} dot={{strokeWidth: 2}} strokeWidth={2} />
+            <Line type="monotone" dataKey="Validation" stroke="#1890ff" activeDot={{ r: 8 }} dot={{ strokeWidth: 2 }} strokeWidth={2} />
         </LineChart>
     </ResponsiveContainer>
 }
@@ -64,9 +71,9 @@ export const ViewLossAccCard = (props) => {
     return <Card
         tabList={LOSS_ACC_TYPES.map((o) => ({ tab: o["label"], key: o["value"] }))}
         activeTabKey={activeTabKey}
-        onTabChange={(key) => {setActiveTabKey(key)}}
+        onTabChange={(key) => { setActiveTabKey(key) }}
     >
-        <ModelLossAccGraph data={lossAccData[activeTabKey]} height = {height} width = {width} yAxisName={LOSS_ACC_TYPES.find((o) => (o.value === activeTabKey))["label"]} />
+        <ModelLossAccGraph data={lossAccData[activeTabKey]} height={height} width={width} yAxisName={LOSS_ACC_TYPES.find((o) => (o.value === activeTabKey))["label"]} />
     </Card>
 }
 
@@ -146,13 +153,13 @@ const ModelCard = (props) => {
     }
 
     return (
-        <div style={{ width: "70%" }}>
+        <div style={modelContainerStyle}>
             <Card title="Model" loading={loadingKeys || loadingArgs || loadingLossAcc}>
                 <Meta />
                 {
                     !loadingKeys && !loadingArgs && (
                         modelKeys ?
-                            <div style={{ textAlign: "left" }}>
+                            <div style={textAlignLeftStyle}>
                                 <Descriptions title="Model Args" bordered column={2}>
                                     {modelArgs && Object.keys(modelArgs).map((key) =>
                                         <Descriptions.Item label={key}>{processModelArg(modelArgs[key])}</Descriptions.Item>
@@ -176,7 +183,7 @@ const ModelCard = (props) => {
                                     lossAccData &&
                                     <>
                                         <Divider />
-                                        <Typography.Title level={5} style={{ paddingBottom: "18px" }}>Model Loss And Accuracy</Typography.Title>
+                                        <Typography.Title level={5} style={titleBottomPaddingStyle}>Model Loss And Accuracy</Typography.Title>
                                         <ViewLossAccCard data={lossAccData} />
                                     </>
                                 }
diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
index 6adb9b8d98..3352e9fdda 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
@@ -16,6 +16,9 @@ import { useLocation } from "react-router-dom";
 import { SocketContext } from "../../../context/socket";
 import { ViewLossAccCard } from "../detail/asset/modelCard";
 
+const contentDivStyle = { padding: "0 24px 0 24px" };
+const paddingBottomSytle = {padding: "0 0 12px 0"};
+
 const PipelineModelVizContent = (props) => {
     const location = useLocation();
     const socket = useContext(SocketContext);
@@ -76,12 +79,12 @@ const PipelineModelVizContent = (props) => {
             modelBids && modelBids !== 404 
             && 
             <div
-                style={{padding: "0 0 12px 0"}}
+                style={paddingBottomSytle}
             >
                 Showing {modelBids.map((bid) => <Tag>{bid}</Tag>)} 
             </div>
         }
-        <div style={{ padding: "0 24px 0 24px" }}>
+        <div style={contentDivStyle}>
             {
                 // show progress bar after getting model batch ids
                 modelBids && modelBids !== 404 && (Object.keys(modelDict).length) !== modelBids.length &&

From cb943d53b650a13b74df379ee233ee84d74227c7 Mon Sep 17 00:00:00 2001
From: Chuxi Wang <wangchuxi94@gmail.com>
Date: Wed, 17 Aug 2022 21:22:34 -0700
Subject: [PATCH 15/17] update graph

---
 .../pipeline/detail/asset/modelCard.js        | 111 ++++++++++++++----
 .../component/pipeline/metaInfo/modelViz.js   |  90 +++++++-------
 .../src/constants/modelVizColors.js           |   0
 3 files changed, 137 insertions(+), 64 deletions(-)
 create mode 100644 droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/constants/modelVizColors.js

diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
index ab17ab9735..11dd2cfdc4 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
@@ -21,7 +21,7 @@ const lineChartMargin = {
     bottom: 5,
 };
 
-const modelContainerStyle = {width: "70%"};
+const modelContainerStyle = { width: "70%" };
 const titleBottomPaddingStyle = { paddingBottom: "18px" };
 const textAlignLeftStyle = { textAlign: "left" };
 
@@ -30,38 +30,91 @@ export const ModelLossAccGraph = (props) => {
     const width = props.width;
     const height = props.height;
     const yAxisName = props.yAxisName;
+    const [plotData, setPlotData] = useState(null);
+
+    const getPlotData = (rawData, bids) => {
+        const getPlotDataPoint = (o, idx, bid) => {
+            const dataPoint = {};
+            dataPoint[`Training_${bid}`] = getScaled(yAxisName, o.training);
+            dataPoint[`Validation_${bid}`] = getScaled(yAxisName, o.validation);
+            dataPoint["Epoch"] = idx;
+            return dataPoint;
+        }
+
+        let plotData = [];
+
+        for (let i = 0; i < bids.length; i++) {
+            const k = bids[i];
+
+            const v = rawData[k];
+            plotData = plotData.concat(v.map((o, idx) => getPlotDataPoint(o, idx, k)));
+
+        }
+
+        return plotData;
+    }
+
+    useEffect(() => {
+        const pd = getPlotData(props.data, props.bids);
+
+        setPlotData(pd);
+    }, [props.bids, props.data]);
 
     const getScaled = (yName, value) => {
         // if is loss, get log2 scaled
         if ((yName) === "Loss") {
             return value ? Math.log2(value) : 0;
         } else {
-            return value;
+            return value ? value : 0;
         }
     }
-    const data = props.data.map((o, idx) => ({ Training: getScaled(yAxisName, o.training), Validation: getScaled(yAxisName, o.validation), Epoch: idx }));
-
     return <ResponsiveContainer aspect={width / height}>
-        <LineChart
-            width={width}
-            height={height}
-            data={data}
-            margin={lineChartMargin}
-        >
-            <CartesianGrid strokeDasharray="3 3" />
-            <XAxis type="number" dataKey="Epoch" label={{ offset: 0, value: "Epoch", position: "insideBottom" }} />
-            <YAxis label={{ offset: 0, value: (yAxisName === "Loss" ? yAxisName + " (Log2 Scaled)" : yAxisName), position: "insideLeft", angle: -90 }} />
-            <Legend />
-            <ChartTooltip />
-            <Line type="monotone" dataKey="Training" stroke="#ad2102" />
-            <Line type="monotone" dataKey="Validation" stroke="#1890ff" activeDot={{ r: 8 }} dot={{ strokeWidth: 2 }} strokeWidth={2} />
-        </LineChart>
+        {plotData &&
+            <LineChart
+                data={plotData}
+                width={width}
+                height={height}
+                margin={lineChartMargin}
+            >
+                <CartesianGrid strokeDasharray="3 3" />
+                <XAxis type="number" dataKey="Epoch" label={{ offset: 0, value: "Epoch", position: "insideBottom" }} />
+                <YAxis label={{ offset: 0, value: (yAxisName === "Loss" ? yAxisName + " (Log2 Scaled)" : yAxisName), position: "insideLeft", angle: -90 }} />
+                <ChartTooltip />
+                <Legend />
+                {
+                    props.bids.map((bid, idx) => {
+                        const alpha =  1 - idx / (props.bids.length * 1.2);
+
+                        return <>
+                            <Line
+                                type="monotone"
+                                key={`Training_${bid}`}
+                                dataKey={`Training_${bid}`}
+                                stroke={`rgba(0, 0, 255, ${alpha})`}
+                            />
+                            <Line
+                                type="monotone"
+                                key={`Validation_${bid}`}
+                                dataKey={`Validation_${bid}`}
+                                stroke={`rgba(255, 0, 0, ${alpha})`}
+                                activeDot={{ r: 8 }}
+                                dot={{ strokeWidth: 2 }}
+                                strokeWidth={2}
+                            />
+                        </>
+
+                    }
+                    )
+                }
+            </LineChart>
+        }
     </ResponsiveContainer>
 }
 
 export const ViewLossAccCard = (props) => {
     const width = props.width ? props.width : 750;
     const height = props.height ? props.height : 400;
+    const bids = props.bids ? props.bids : null;
     const lossAccData = props.data;
 
     const LOSS_ACC_TYPES = [{ "label": "Accuracy", "value": "acc" }, { "label": "Loss", "value": "loss" }]
@@ -73,7 +126,12 @@ export const ViewLossAccCard = (props) => {
         activeTabKey={activeTabKey}
         onTabChange={(key) => { setActiveTabKey(key) }}
     >
-        <ModelLossAccGraph data={lossAccData[activeTabKey]} height={height} width={width} yAxisName={LOSS_ACC_TYPES.find((o) => (o.value === activeTabKey))["label"]} />
+        <ModelLossAccGraph
+            data={lossAccData[activeTabKey]}
+            height={height} width={width}
+            yAxisName={LOSS_ACC_TYPES.find((o) => (o.value === activeTabKey))["label"]}
+            bids={bids}
+        />
     </Card>
 }
 
@@ -85,7 +143,10 @@ const ModelCard = (props) => {
     const [loadingKeys, setLoadingKeys] = useState(true);
     const [currentModelKey, setCurrentModelKey] = useState(null);
     const [attrModalOpen, setAttrModalOpen] = useState(false);
-    const [lossAccData, setLossAccData] = useState(null);
+    const [lossAccData, setLossAccData] = useState({
+        loss: {},
+        acc: {}
+    });
     const [loadingLossAcc, setLoadingLossAcc] = useState(true);
 
     const socket = useContext(SocketContext);
@@ -108,7 +169,11 @@ const ModelCard = (props) => {
     const handleReceivedLossAcc = useCallback((data) => {
         setLoadingLossAcc(false);
         if (data !== 404) {
-            setLossAccData(data[0]);
+            setLossAccData((prev) => ({
+                ...prev,
+                loss: { ...prev.loss, [data[1]]: data[0].loss },
+                acc: { ...prev.acc, [data[1]]: data[0].acc },
+            }));
         }
     });
 
@@ -184,7 +249,7 @@ const ModelCard = (props) => {
                                     <>
                                         <Divider />
                                         <Typography.Title level={5} style={titleBottomPaddingStyle}>Model Loss And Accuracy</Typography.Title>
-                                        <ViewLossAccCard data={lossAccData} />
+                                        <ViewLossAccCard data={lossAccData} bids={[parseInt(batchId)]} />
                                     </>
                                 }
                             </div>
@@ -208,4 +273,4 @@ const ModelCard = (props) => {
         </div>);
 }
 
-export default ModelCard;
\ No newline at end of file
+export default ModelCard;
diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
index 3352e9fdda..5c4c8b3938 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
@@ -10,54 +10,51 @@ Usage:
 */
 
 import { SyncOutlined } from "@ant-design/icons";
-import { Progress, Tag, Typography } from "antd";
+import { Progress, Spin, Tag, Typography } from "antd";
 import React, { useContext, useEffect, useState } from "react";
 import { useLocation } from "react-router-dom";
 import { SocketContext } from "../../../context/socket";
 import { ViewLossAccCard } from "../detail/asset/modelCard";
 
 const contentDivStyle = { padding: "0 24px 0 24px" };
-const paddingBottomSytle = {padding: "0 0 12px 0"};
+const paddingBottomSytle = { padding: "0 0 12px 0" };
+const { CheckableTag } = Tag;
 
 const PipelineModelVizContent = (props) => {
     const location = useLocation();
     const socket = useContext(SocketContext);
     const [modelBids, setModelBids] = useState(null);
+    const [selectedBids, setSelectedBids] = useState(null);
     const [currentIdx, setCurrentIdx] = useState(0);
-    const [modelDict, ] = useState({});
-    const [modelCombined, setModelCombined] = useState(null);
+    const [modelDict, setModelDict] = useState({
+        loss: {},
+        acc: {}
+    });
 
     const handleReceivedModelBids = (data) => {
-        setModelBids(data.map((bid) => parseInt(bid)));
+        setModelBids(data);
+        setSelectedBids(data);
     }
 
     const handleReceivedModelLossAcc = (data) => {
-        if (data !== 404) {
-            modelDict[data[1]] = data[0];
-
-            if (modelBids && modelBids !== 404 && Object.keys(modelDict).length < modelBids.length) {
-                setCurrentIdx(Object.keys(modelDict).length);
-            } else if (modelBids && modelBids !== 404 && Object.keys(modelDict).length === modelBids.length) {
-                // finished loading, combine based on bid sequence
-                const modelArr = modelBids.map((bid) => (modelDict[bid]));
-                let comb = {}
-
-                for (let i = 0; i < modelArr.length; i++) {
-                    const o = modelArr[i];
-                    for (let j = 0; j < Object.keys(o).length; j++) {
-                        const key = Object.keys(o)[j];
-                        if (!(key in comb)) {
-                            comb[key] = o[key];
-                        } else {
-                            comb[key] = [...comb[key], ...o[key]]
-                        }
-                    }
-                }
-                setModelCombined(comb);
-            }
+        if (data !== 404 && modelBids && modelBids !== 404 && !(data[1] in modelDict.loss)) {
+            setModelDict((prevModelDict) => ({
+                ...prevModelDict,
+                loss: {...prevModelDict.loss, [data[1]]: data[0].loss},
+                acc: {...prevModelDict.acc, [data[1]]: data[0].acc},
+            }));
+            setCurrentIdx(currentIdx + 1);
         }
     }
 
+    const handleToogleCheckedBid = (bid, checked) => {
+        // toggle checked for the selected bid
+        const nextSelectedBids = checked
+            ? [...selectedBids, bid]
+            : selectedBids.filter((b) => b !== bid);
+        setSelectedBids(nextSelectedBids);
+    }
+
     useEffect(() => {
         socket.on("get_best_model_bids_by_pipeline", (data) => handleReceivedModelBids(data));
         socket.on("get_best_model_loss_acc_by_id", (data) => handleReceivedModelLossAcc(data));
@@ -68,7 +65,7 @@ const PipelineModelVizContent = (props) => {
     }, []);
 
     useEffect(() => {
-        if (modelBids && modelBids !== 404 && currentIdx >= 0) {
+        if (modelBids && modelBids !== 404 && currentIdx >= 0 && currentIdx < modelBids.length) {
             socket.emit("get_best_model_loss_acc_by_id", modelBids[currentIdx]);
         }
     }, [modelBids, currentIdx]);
@@ -76,36 +73,47 @@ const PipelineModelVizContent = (props) => {
     return <div>
         <Typography.Title level={4}>{"View Model Accuracy & Loss"}</Typography.Title>
         {
-            modelBids && modelBids !== 404 
-            && 
+            (!modelBids || !selectedBids) && <Spin />
+        }
+        {
+            modelBids && modelBids !== 404 && selectedBids && selectedBids !== 404
+            &&
             <div
                 style={paddingBottomSytle}
             >
-                Showing {modelBids.map((bid) => <Tag>{bid}</Tag>)} 
+                Showing {modelBids.map((bid) =>
+                    <CheckableTag
+                        key={bid}
+                        checked={selectedBids.includes(bid)}
+                        onChange={(checked) => handleToogleCheckedBid(bid, checked)}
+                    >
+                        {bid}
+                    </CheckableTag>
+                )}
             </div>
         }
         <div style={contentDivStyle}>
             {
                 // show progress bar after getting model batch ids
-                modelBids && modelBids !== 404 && (Object.keys(modelDict).length) !== modelBids.length &&
-                <div> 
+                modelBids && modelBids !== 404 && currentIdx !== modelBids.length &&
+                <div>
                     <span><SyncOutlined spin /> Loading Model Data... </span>
-                    <Progress percent={Math.trunc((Object.keys(modelDict).length) / modelBids.length * 100)} />
+                    <Progress percent={Math.trunc((currentIdx) / modelBids.length * 100)} />
                 </div>
             }
 
             {
                 // show graph when loading is done
-                modelBids && modelBids !== 404 && (Object.keys(modelDict).length) === modelBids.length && modelCombined &&
+                modelBids && modelBids !== 404 && currentIdx === modelBids.length &&
                 <div>
-                    <ViewLossAccCard data={modelCombined} width={1500}/>
+                    <ViewLossAccCard data={modelDict} width={1500} bids={selectedBids} />
                 </div>
-            } 
+            }
             {
                 // show no data when not having any batch ids
-                modelBids && modelBids === 404 && 
+                modelBids && modelBids === 404 &&
                 <div>
-                    Sorry, no model data is avaible yet. 
+                    Sorry, no model data is avaible yet.
                 </div>
             }
         </div>
@@ -113,4 +121,4 @@ const PipelineModelVizContent = (props) => {
     </div>
 }
 
-export default PipelineModelVizContent;
\ No newline at end of file
+export default PipelineModelVizContent;
diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/constants/modelVizColors.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/constants/modelVizColors.js
new file mode 100644
index 0000000000..e69de29bb2

From b7d611af0f4070f47b6d0cea4e4976340d8edd4c Mon Sep 17 00:00:00 2001
From: Chuxi Wang <wangchuxi94@gmail.com>
Date: Wed, 17 Aug 2022 21:40:59 -0700
Subject: [PATCH 16/17] add style fix

---
 .../pipeline/detail/asset/modelCard.js        |  8 +++--
 .../component/pipeline/metaInfo/modelViz.js   |  2 +-
 .../src/constants/modelVizColors.js           | 35 +++++++++++++++++++
 3 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
index 11dd2cfdc4..3672c3baa5 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js
@@ -5,12 +5,14 @@ The card showing Model infomation of a run.
 
 Usage:
 <ModelCard batchId = {batchId} pipelineType={pipelineType}/>
+<ModelLossAccGraph width={width} height={height} bids={batchIds (list)} data={data}>
 */
 import { Button, Card, Descriptions, Divider, Tooltip, Typography } from "antd";
 import React, { useCallback, useContext, useEffect, useState } from "react";
 import { SocketContext } from "../../../../context/socket";
 import ModelAtrributeModal from "./modelAttributeDetailModal";
 import { LineChart, Line, XAxis, YAxis, CartesianGrid, Legend, Tooltip as ChartTooltip, ResponsiveContainer } from "recharts";
+import {BLUE_GREEN_COLS, RED_ORG_COLS} from "../../../../constants/modelVizColors";
 
 const { Meta } = Card;
 
@@ -83,20 +85,20 @@ export const ModelLossAccGraph = (props) => {
                 <Legend />
                 {
                     props.bids.map((bid, idx) => {
-                        const alpha =  1 - idx / (props.bids.length * 1.2);
+                        const colIdx =  idx % props.bids.length;
 
                         return <>
                             <Line
                                 type="monotone"
                                 key={`Training_${bid}`}
                                 dataKey={`Training_${bid}`}
-                                stroke={`rgba(0, 0, 255, ${alpha})`}
+                                stroke={RED_ORG_COLS[colIdx]}
                             />
                             <Line
                                 type="monotone"
                                 key={`Validation_${bid}`}
                                 dataKey={`Validation_${bid}`}
-                                stroke={`rgba(255, 0, 0, ${alpha})`}
+                                stroke={BLUE_GREEN_COLS[colIdx]}
                                 activeDot={{ r: 8 }}
                                 dot={{ strokeWidth: 2 }}
                                 strokeWidth={2}
diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
index 5c4c8b3938..7695a826da 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
@@ -106,7 +106,7 @@ const PipelineModelVizContent = (props) => {
                 // show graph when loading is done
                 modelBids && modelBids !== 404 && currentIdx === modelBids.length &&
                 <div>
-                    <ViewLossAccCard data={modelDict} width={1500} bids={selectedBids} />
+                    <ViewLossAccCard data={modelDict} width={1200} height={800} bids={selectedBids} />
                 </div>
             }
             {
diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/constants/modelVizColors.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/constants/modelVizColors.js
index e69de29bb2..dc7b421fa2 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/constants/modelVizColors.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/constants/modelVizColors.js
@@ -0,0 +1,35 @@
+export const BLUE_GREEN_COLS = [
+    "#1d39c4",
+    "#597ef7",
+    "#adc6ff",
+
+    "#096dd9",
+    "#40a9ff",
+    "#91d5ff",
+
+    "#08979c",
+    "#36cfc9",
+    "#87e8de",
+    
+    "#389e0d",
+    "#73d13d",
+    "#b7eb8f",
+]
+
+export const RED_ORG_COLS = [
+    "#cf1322",
+    "#ff4d4f",
+    "#ffa39e",
+
+    "#d4380d",
+    "#ff7a45",
+    "#ffbb96",
+
+    "#d46b08",
+    "#ffa940",
+    "#ffd591",
+
+    "#d4b106",
+    "#ffec3d",
+    "#fffb8f",
+]

From 24fc4213e24c2feda8849af2dd5e9d69d10f7a3b Mon Sep 17 00:00:00 2001
From: Chuxi Wang <wangchuxi94@gmail.com>
Date: Wed, 17 Aug 2022 22:01:05 -0700
Subject: [PATCH 17/17] changed style

---
 .../src/component/pipeline/metaInfo/modelViz.js                 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
index 7695a826da..20ea283b16 100644
--- a/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
+++ b/droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/metaInfo/modelViz.js
@@ -106,7 +106,7 @@ const PipelineModelVizContent = (props) => {
                 // show graph when loading is done
                 modelBids && modelBids !== 404 && currentIdx === modelBids.length &&
                 <div>
-                    <ViewLossAccCard data={modelDict} width={1200} height={800} bids={selectedBids} />
+                    <ViewLossAccCard data={modelDict} width={900} bids={selectedBids} />
                 </div>
             }
             {