Hitl dashbaord model viz improve

droidlet/tools/hitl/dashboard_app/README.MD

@@ -1,9 +1,18 @@
 # Dashboard App for HITL
-Updated July 22 by Chuxi.
+Updated Aug 4 by Chuxi.
 
 This is a Dashboard app prototype for the HITL system.
 
 ## Update Note
+- Aug 4:
+    - Update support for visualizing the model:
+        - Changed from showing loss and accuracy of validation and text_span to showing loss and accuracy separately, and show training loss/accuracy & validation loss/accuracy in the same graph. 
+        - Updated the graph visualization container to be responsive to browser window changes. 
+        - Demo:
+            - Best models loss & accuracy for a pipeline:
+                - ![demo_best_models_for_pipeline](https://user-images.githubusercontent.com/51009396/182976894-a1eb380b-06fd-491b-8e4f-29abd7c2024d.gif)
+            - Best model in a run (showing loss and accuracy seperately, graph responsive to browser window changes):
+                - ![demo_best_model_for_a_run](https://user-images.githubusercontent.com/51009396/182976905-60713821-31d8-4187-889a-11b3b1d1ce82.gif)
 - July 22:
     - Updated frontend component to show the model line graph of loss and accuracy vs epoch.
     - Demo:
@@ -128,15 +137,22 @@ APIs are based on socket event, the following APIs are supported currently:
         - the key for the model, could be any key from the model, or "COMPLETE", indicating getting the complete model dict
     - output: the key and the value specific to the key for the model if the model exists and key is valid, otherwise error code
 - get_best_model_loss_acc_by_id
-    - get loss and accuracy from a model log for a specific run's best model, 
-    the loss and accuracy infomation is reterived by crawling the best model's log file
+    - get loss and accuracy from a model log for a specific run's best model, the loss and accuracy infomation is reterived by crawling the best model's log file
     - input:
         - batch id of a specific run
     - output:
-        - a dictionary containing:
-            - epoch loss and accuracy
-            - text_span loss and accuracy
+        - a list contains the following when a best model log file can be found:
+            - a dictionary containing:
+                - key: loss, value: loss list for training and validation dataset
+                - key: acc, value: accuracy list for training and validation dataset
+            - batch id 
         - or an error code indicating the best model log cannot be find
+- get_best_model_bids_by_pipeline
+    - get best models' batch ids for a specific pipeline
+    - input:
+        - pipeline name (can be nlu, tao or vision)
+    - output:
+        - a list of the batch ids of the best models
 
 ## Demo
 ![backend_api_demo](https://user-images.githubusercontent.com/51009396/175696481-532cec55-5b2e-4bae-bceb-9e7d3f2aa7b7.gif)

droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py

@@ -199,7 +199,7 @@ def get_best_model_loss_acc_by_id(batch_id: int):
     """
     Get best model loss and accuracy from model log file,
     return:
-        - a dict including epoch & text span loss and accuracy, and no error code if can find the best model log
+        - a dict including loss and accuracy for training and valiation, and no error code if can find the best model log
         - an error message with error code 404 if cannot find the best model log
     """
 
@@ -228,5 +228,20 @@ def get_best_model_name(batch_id: int):
         return f"Cannot find best model log file with batch_id = {batch_id}", 404
 
     # read best model log file
-    epoch_ls, text_span_ls = read_model_log_to_list(best_model_log_fname)
-    return {"epoch": epoch_ls, "text_span": text_span_ls}, None
+    loss_ls, acc_ls = read_model_log_to_list(best_model_log_fname)
+    print(f"loss length: {len(loss_ls)}, accuracy length: {len(acc_ls)}")
+    return {"loss": loss_ls, "acc": acc_ls}, None
+
+
+def get_best_models_by_pipeline(pipeline: str):
+    """
+    Get best models batch_id list for a pipeline.
+    The pipeline can be nlu, tao or vision
+    """
+    local_fname = _download_file(f"{pipeline}_model_viz_batch_list.txt")
+    if local_fname is None:
+        return f"Cannot find best model list for {pipeline}", 404
+
+    model_batch_id_list = _read_file(local_fname).split("\n")
+
+    return model_batch_id_list, None

droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py

@@ -10,6 +10,7 @@
 import json
 from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import (
     get_best_model_loss_acc_by_id,
+    get_best_models_by_pipeline,
     get_dataset_by_name,
     get_dataset_indices_by_id,
     get_dataset_version_list_by_pipeline,
@@ -50,6 +51,7 @@ class DASHBOARD_EVENT(Enum):
     GET_MODEL_KEYS = "get_model_keys_by_id"
     GET_MODEL_VALUE = "get_model_value_by_id_n_key"
     GET_BEST_MODEL_LOSS_ACC = "get_best_model_loss_acc_by_id"
+    GET_BEST_MODELS = "get_best_model_bids_by_pipeline"
 
 
 # constants for model related apis
@@ -235,9 +237,11 @@ def get_best_model_loss_acc(batch_id: int):
     - input:
         - batch id of a specific run
     - output:
-        - a dictionary containing:
-            - epoch loss and accuracy
-            - text_span loss and accuracy
+        - a list contains the following when a best model log file can be found:
+            - a dictionary containing:
+                - key: loss, value: loss list for training and validation dataset
+                - key: acc, value: accuracy list for training and validation dataset
+            - batch id
         - or an error code indicating the best model log cannot be find
     """
     print(
@@ -247,7 +251,24 @@ def get_best_model_loss_acc(batch_id: int):
     if error_code:
         emit(DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value, error_code)
     else:
-        emit(DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value, loss_acc_dict)
+        emit(DASHBOARD_EVENT.GET_BEST_MODEL_LOSS_ACC.value, [loss_acc_dict, int(batch_id)])
+
+
+@socketio.on(DASHBOARD_EVENT.GET_BEST_MODELS.value)
+def get_best_model_batches(pipeline: str):
+    """
+    get best models' batch ids for a specific pipeline
+    - input:
+        - pipeline name (can be nlu, tao or vision)
+    - output:
+        - a list of the batch ids of the best models
+    """
+    print(f"Request received: {DASHBOARD_EVENT.GET_BEST_MODELS.value}, pipeline = {pipeline}")
+    model_dict, error_code = get_best_models_by_pipeline(pipeline)
+    if error_code:
+        emit(DASHBOARD_EVENT.GET_BEST_MODELS.value, error_code)
+    else:
+        emit(DASHBOARD_EVENT.GET_BEST_MODELS.value, model_dict)
 
 
 if __name__ == "__main__":

droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js

@@ -5,65 +5,150 @@ The card showing Model infomation of a run.
 
 Usage:
 <ModelCard batchId = {batchId} pipelineType={pipelineType}/>
+<ModelLossAccGraph width={width} height={height} bids={batchIds (list)} data={data}>
 */
 import { Button, Card, Descriptions, Divider, Tooltip, Typography } from "antd";
 import React, { useCallback, useContext, useEffect, useState } from "react";
 import { SocketContext } from "../../../../context/socket";
 import ModelAtrributeModal from "./modelAttributeDetailModal";
-import { LineChart, Line, XAxis, YAxis, CartesianGrid, Legend, Tooltip as ChartTooltip } from 'recharts';
+import { LineChart, Line, XAxis, YAxis, CartesianGrid, Legend, Tooltip as ChartTooltip, ResponsiveContainer } from "recharts";
+import {BLUE_GREEN_COLS, RED_ORG_COLS} from "../../../../constants/modelVizColors";
 
 const { Meta } = Card;
-const LOSS_ACC_TYPES = [{ "label": "Epoch", "value": "epoch" }, { "label": "Text Span", "value": "text_span" }]
-
-const ModelLossAccGraph = (props) => {
-    const data = props.data.map((o, idx) => ({ Loss: o.loss, Accuracy: o.acc, Epoch: idx }));
-
-    return <div style={{ width: "100%", height: "100%" }}>
-        <LineChart
-            width={750}
-            height={400}
-            data={data}
-            margin={{
-                top: 5,
-                right: 30,
-                left: 20,
-                bottom: 5,
-              }}
-        >
-            <CartesianGrid strokeDasharray="3 3" />
-            <XAxis type="number" dataKey="Epoch" label={{offset: 0, value: "Epoch", position: "insideBottom" }}/>
-            <YAxis />
-            <Legend />
-            <ChartTooltip />
-            <Line type="monotone" dataKey="Loss" stroke="#ad2102" />
-            <Line type="monotone" dataKey="Accuracy" stroke="#1890ff" activeDot={{ r: 8 }} dot={{strokeWidth: 2}} strokeWidth={2} />
-        </LineChart>
-    </div>
+
+const lineChartMargin = {
+    top: 5,
+    right: 30,
+    left: 20,
+    bottom: 5,
+};
+
+const modelContainerStyle = { width: "70%" };
+const titleBottomPaddingStyle = { paddingBottom: "18px" };
+const textAlignLeftStyle = { textAlign: "left" };
+
+
+export const ModelLossAccGraph = (props) => {
+    const width = props.width;
+    const height = props.height;
+    const yAxisName = props.yAxisName;
+    const [plotData, setPlotData] = useState(null);
+
+    const getPlotData = (rawData, bids) => {
+        const getPlotDataPoint = (o, idx, bid) => {
+            const dataPoint = {};
+            dataPoint[`Training_${bid}`] = getScaled(yAxisName, o.training);
+            dataPoint[`Validation_${bid}`] = getScaled(yAxisName, o.validation);
+            dataPoint["Epoch"] = idx;
+            return dataPoint;
+        }
+
+        let plotData = [];
+
+        for (let i = 0; i < bids.length; i++) {
+            const k = bids[i];
+
+            const v = rawData[k];
+            plotData = plotData.concat(v.map((o, idx) => getPlotDataPoint(o, idx, k)));
+
+        }
+
+        return plotData;
+    }
+
+    useEffect(() => {
+        const pd = getPlotData(props.data, props.bids);
+
+        setPlotData(pd);
+    }, [props.bids, props.data]);
+
+    const getScaled = (yName, value) => {
+        // if is loss, get log2 scaled
+        if ((yName) === "Loss") {
+            return value ? Math.log2(value) : 0;
+        } else {
+            return value ? value : 0;
+        }
+    }
+    return <ResponsiveContainer aspect={width / height}>
+        {plotData &&
+            <LineChart
+                data={plotData}
+                width={width}
+                height={height}
+                margin={lineChartMargin}
+            >
+                <CartesianGrid strokeDasharray="3 3" />
+                <XAxis type="number" dataKey="Epoch" label={{ offset: 0, value: "Epoch", position: "insideBottom" }} />
+                <YAxis label={{ offset: 0, value: (yAxisName === "Loss" ? yAxisName + " (Log2 Scaled)" : yAxisName), position: "insideLeft", angle: -90 }} />
+                <ChartTooltip />
+                <Legend />
+                {
+                    props.bids.map((bid, idx) => {
+                        const colIdx =  idx % props.bids.length;
+
+                        return <>
+                            <Line
+                                type="monotone"
+                                key={`Training_${bid}`}
+                                dataKey={`Training_${bid}`}
+                                stroke={RED_ORG_COLS[colIdx]}
+                            />
+                            <Line
+                                type="monotone"
+                                key={`Validation_${bid}`}
+                                dataKey={`Validation_${bid}`}
+                                stroke={BLUE_GREEN_COLS[colIdx]}
+                                activeDot={{ r: 8 }}
+                                dot={{ strokeWidth: 2 }}
+                                strokeWidth={2}
+                            />
+                        </>
+
+                    }
+                    )
+                }
+            </LineChart>
+        }
+    </ResponsiveContainer>
 }
 
-const ViewLossAccCard = (props) => {
+export const ViewLossAccCard = (props) => {
+    const width = props.width ? props.width : 750;
+    const height = props.height ? props.height : 400;
+    const bids = props.bids ? props.bids : null;
     const lossAccData = props.data;
+
+    const LOSS_ACC_TYPES = [{ "label": "Accuracy", "value": "acc" }, { "label": "Loss", "value": "loss" }]
+
     const [activeTabKey, setActiveTabKey] = useState(LOSS_ACC_TYPES[0]["value"]);
 
     return <Card
         tabList={LOSS_ACC_TYPES.map((o) => ({ tab: o["label"], key: o["value"] }))}
         activeTabKey={activeTabKey}
-        onTabChange={(key) => setActiveTabKey(key)}
+        onTabChange={(key) => { setActiveTabKey(key) }}
     >
-        <ModelLossAccGraph data={lossAccData[activeTabKey]} />
+        <ModelLossAccGraph
+            data={lossAccData[activeTabKey]}
+            height={height} width={width}
+            yAxisName={LOSS_ACC_TYPES.find((o) => (o.value === activeTabKey))["label"]}
+            bids={bids}
+        />
     </Card>
 }
 
 const ModelCard = (props) => {
     const batchId = props.batchId;
-    const pipelineType = props.pipelineType;
     const [modelArgs, setModelArgs] = useState(null);
     const [modelKeys, setModelKeys] = useState(null);
     const [loadingArgs, setLoadingArgs] = useState(true);
     const [loadingKeys, setLoadingKeys] = useState(true);
     const [currentModelKey, setCurrentModelKey] = useState(null);
     const [attrModalOpen, setAttrModalOpen] = useState(false);
-    const [lossAccData, setLossAccData] = useState(null);
+    const [lossAccData, setLossAccData] = useState({
+        loss: {},
+        acc: {}
+    });
     const [loadingLossAcc, setLoadingLossAcc] = useState(true);
 
     const socket = useContext(SocketContext);
@@ -86,7 +171,11 @@ const ModelCard = (props) => {
     const handleReceivedLossAcc = useCallback((data) => {
         setLoadingLossAcc(false);
         if (data !== 404) {
-            setLossAccData(data);
+            setLossAccData((prev) => ({
+                ...prev,
+                loss: { ...prev.loss, [data[1]]: data[0].loss },
+                acc: { ...prev.acc, [data[1]]: data[0].acc },
+            }));
         }
     });
 
@@ -130,18 +219,14 @@ const ModelCard = (props) => {
         setAttrModalOpen(true);
     }
 
-    const handleViewModelLossAndAcc = (lossAccType) => {
-        alert(lossAccData[lossAccType].map((o) => (`loss: ${o.loss},  acc: ${o.acc}`)));
-    }
-
     return (
-        <div style={{ width: '70%' }}>
+        <div style={modelContainerStyle}>
             <Card title="Model" loading={loadingKeys || loadingArgs || loadingLossAcc}>
                 <Meta />
                 {
                     !loadingKeys && !loadingArgs && (
                         modelKeys ?
-                            <div style={{ textAlign: "left" }}>
+                            <div style={textAlignLeftStyle}>
                                 <Descriptions title="Model Args" bordered column={2}>
                                     {modelArgs && Object.keys(modelArgs).map((key) =>
                                         <Descriptions.Item label={key}>{processModelArg(modelArgs[key])}</Descriptions.Item>
@@ -165,16 +250,16 @@ const ModelCard = (props) => {
                                     lossAccData &&
                                     <>
                                         <Divider />
-                                        <Typography.Title level={5} style={{ paddingBottom: '18px' }}>Model Loss And Accuracy</Typography.Title>
-                                        <ViewLossAccCard data={lossAccData} />
+                                        <Typography.Title level={5} style={titleBottomPaddingStyle}>Model Loss And Accuracy</Typography.Title>
+                                        <ViewLossAccCard data={lossAccData} bids={[parseInt(batchId)]} />
                                     </>
                                 }
                             </div>
                             :
                             <div>NA</div>
                     )
                 }
-                {/* modal showing a specific model attribute's field (anything other than args) */}
+                {/* modal showing a specific model attribute"s field (anything other than args) */}
                 {currentModelKey
                     &&
                     <ModelAtrributeModal
@@ -190,4 +275,4 @@ const ModelCard = (props) => {
         </div>);
 }
 
-export default ModelCard;
+export default ModelCard;