Merge pull request #13 from TurboTelescope/notes

docs

components/LogViewer.tsx

@@ -3,10 +3,10 @@
 import { Result, Rx, useRxSet, useRxSuspenseSuccess } from "@effect-rx/rx-react";
 import { FetchHttpClient, HttpClient } from "@effect/platform";
 import { Effect, Stream } from "effect";
+import { useMemo } from "react";
 
 import { rpcClient } from "@/app/api/client";
 import { SchemaName, VerboseLogRequest, VerboseLogURLRequest } from "@/services/Domain";
-import { useMemo } from "react";
 
 const runtime = Rx.runtime(FetchHttpClient.layer);
 

components/PipelineHealth.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import { Result, useRx, useRxSet, useRxValue } from "@effect-rx/rx-react";
-import { DateTime, Duration, Effect } from "effect";
+import { DateTime } from "effect";
 import { Suspense, useMemo } from "react";
 
 import { AggregateBySelector } from "@/components/PipelineHealth/AggregateBySelector";
@@ -22,15 +22,6 @@ export function PipelineHealth() {
     const pullTimeSeriesData = useRxSet(timeSeriesGroupedRx);
     useMemo(pullTimeSeriesData, [pullTimeSeriesData]);
 
-    const updateFrom = useRxSet(fromRx);
-    useMemo(
-        () => updateFrom(Effect.runSync(DateTime.now).pipe(DateTime.subtractDuration(Duration.days(3)))),
-        [updateFrom]
-    );
-
-    const updateUntil = useRxSet(untilRx);
-    useMemo(() => updateUntil(Effect.runSync(DateTime.now)), [updateUntil]);
-
     // Gets
     const from = useRxValue(fromRx);
     const until = useRxValue(untilRx);

components/PipelineHealth/AggregateBySelector.tsx

@@ -1,3 +1,26 @@
+/**
+ * Aggregation determines at what interval the time series data is grouped by.
+ * The time series data can be grouped by seconds, minutes, hours, days, months,
+ * or years. The AggregateBySelector component is a dropdown menu that allows
+ * the user to select the aggregation interval.
+ *
+ * Aggregation by seconds and minutes is disabled when include empty buckets is
+ * enabled because otherwise the graph would be too dense / render poorly. The
+ * database service only returns the raw data from the database and we apply
+ * this transformation and empty buckets on the client so that we don't have to
+ * refetch data when the user changes this selection.
+ *
+ * TODO: I think aggregation by seconds and minutes does make sense so long as
+ * the time range is small enough. A quantifiable metric might be the number of
+ * resultant buckets, i.e if selecting seconds would result in more than N
+ * buckets (where N is a threshold determined by testing until the graph renders
+ * poorly) then disable it. This metric should be implemented by pulling in the
+ * RowsRx, and taking all the keys from that map. The grouping function from the
+ * RowsRx should be pulled out so that it could be reused here, and then the
+ * same grouping logic should be applied here for each aggregation method,
+ * disabling all aggregation methods that exceed the threshold.
+ */
+
 "use client";
 
 import { useRx, useRxValue } from "@effect-rx/rx-react";

components/PipelineHealth/EmptyBucketsToggle.tsx

@@ -1,3 +1,24 @@
+/**
+ * Empty buckets determines whether or not to show buckets that have no data.
+ * This is useful for showing gaps in data, but is disabled by default for
+ * performance reasons.
+ *
+ * When empty buckets are enable, after the data is aggregated, we will fill in
+ * any missing buckets with empty data. The database service only returns the
+ * raw data from the database and we apply this transformation and aggregation
+ * on the client so that we don't have to refetch data when the user changes
+ * this selection.
+ *
+ * If aggregation by seconds or minutes is enabled, empty buckets will be
+ * disabled because the graph would be too dense and render poorly.
+ *
+ * TODO: I think showing empty buckets does make sense so long as the number of
+ * buckets does not exceed a threshold that would cause the graph to render
+ * poorly. This threshold could be determined through testing. A quantifiable
+ * metric would be the number of buckets after aggregation is applied. See the
+ * comment in AggregateBySelector.tsx for more details.
+ */
+
 "use client";
 
 import { useRx, useRxValue } from "@effect-rx/rx-react";

components/PipelineHealth/LocaleSelector.tsx

@@ -1,3 +1,23 @@
+/**
+ * When I started the image health website, I used "locale" when what I actually
+ * meant was "timezone." TODO: replace "locale" with "timezone" throughout the
+ * app.
+ *
+ * This component is a dropdown menu that allows the user to select a timezone.
+ * By default, the timezone should be set to the timezone of the user's device.
+ * Any times referenced in the app (outside of updateFromRx and updateUntilRx)
+ * should be in this timezone.
+ *
+ * The exceptions for updateFromRx and updateUntilRx are intentional. When
+ * setting a new time for the range, you can give it any time in any time range.
+ * The timezone will then be applied to the datetime without changing the time
+ * itself. This approach has serval advantages over tracking everything as a UTC
+ * datetime or a Zoned datetime or a Local datetime. And not needing to make
+ * sure that the timezone is correct when setting a new from/until rx is
+ * important because those times should always come from the calendar and it is
+ * easy to forget to offset for the timezone otherwise.
+ */
+
 "use client";
 
 import { Result, useRx } from "@effect-rx/rx-react";

components/PipelineHealth/PipelineStepHistogram.tsx

@@ -1,13 +1,13 @@
 "use client";
 
+import { useRxSuspenseSuccess } from "@effect-rx/rx-react";
 import { Array, Function, Option, Record, Schema, Tuple } from "effect";
 import { Bar, BarChart, CartesianGrid, XAxis, YAxis } from "recharts";
 
+import { rowsRx } from "@/components/PipelineHealth/rx";
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
 import { ChartConfig, ChartContainer, ChartTooltip, ChartTooltipContent } from "@/components/ui/chart";
 import { PipelineStepName, ShortPipelineName } from "@/services/Domain";
-import { useRxSuspenseSuccess } from "@effect-rx/rx-react";
-import { rowsRx } from "./rx";
 
 const chart1 = "percentPipelineFailure" as const;
 
@@ -44,6 +44,7 @@ export function PipelineStepHistogram() {
             return group;
         })
     );
+
     const sorted = bucketsWithFailures.sort(
         (a, b) => PipelineStepName.literals.indexOf(a.pipelineStep) - PipelineStepName.literals.indexOf(b.pipelineStep)
     );
@@ -72,9 +73,9 @@ export function PipelineStepHistogram() {
                             tickLine={true}
                             axisLine={false}
                             tickMargin={8}
-                            tickFormatter={(longname: typeof PipelineStepName.Type) => {
-                                return Schema.decodeSync(ShortPipelineName)(longname);
-                            }}
+                            tickFormatter={(longName: typeof PipelineStepName.Type) =>
+                                Schema.decodeSync(ShortPipelineName)(longName)
+                            }
                         />
                         <YAxis tickLine={true} axisLine={false} tickMargin={8} tickFormatter={(value) => `${value}`} />
                         <ChartTooltip content={<ChartTooltipContent className="w-[275px]" />} />

components/PipelineHealth/RunTimeHist.tsx

@@ -43,7 +43,7 @@ export const chartConfigs = {
     },
     [chart5]: {
         color: "#FF0000",
-        title: "Number of Failed of Runs",
+        title: "Number of Failed Runs",
         label: "Number of Failed Runs",
         icon: Cross2Icon,
     },
@@ -101,7 +101,8 @@ export function AverageProcessingTimeLineChart() {
             })
         )
     );
-    const activeChartKey = activeChart === "All" ? chart6 : activeChart === "success" ? chart2 : chart3;
+    const activeChartKey = activeChart === "all" ? chart6 : activeChart === "success" ? chart2 : chart3;
+
     // Chart implementation
     return (
         <Card>
@@ -121,7 +122,7 @@ export function AverageProcessingTimeLineChart() {
                                 className="flex flex-1 flex-col justify-center gap-1 border-t px-6 py-4 text-left even:border-l data-[active=true]:bg-muted/50 sm:border-l sm:border-t-0 sm:px-8 sm:py-6"
                                 onClick={() =>
                                     chart === "averageProcessingTimeAllRuns"
-                                        ? setActiveChart("All")
+                                        ? setActiveChart("all")
                                         : chart === "averageSuccessProcessingTime"
                                           ? setActiveChart("success")
                                           : setActiveChart("failure")
@@ -201,7 +202,7 @@ export function AverageProcessingTimeLineChart() {
                             height={36}
                         />
 
-                        {activeChart === "All" ? (
+                        {activeChart === "all" ? (
                             <Bar
                                 key={`${chart6}.${chart2}-bar`}
                                 dataKey={`${chart2}`}
@@ -212,7 +213,7 @@ export function AverageProcessingTimeLineChart() {
                         ) : (
                             <></>
                         )}
-                        {activeChart === "All" ? (
+                        {activeChart === "all" ? (
                             <Bar
                                 key={`${chart6}.${chart3}-bar`}
                                 dataKey={`${chart3}`}
@@ -224,7 +225,7 @@ export function AverageProcessingTimeLineChart() {
                             <></>
                         )}
 
-                        {activeChart !== "All" ? (
+                        {activeChart !== "all" ? (
                             <Bar
                                 key={activeChart === "success" ? `${chart2}-bar` : `${chart3}-bar`}
                                 dataKey={activeChart === "success" ? chart2 : chart3}

components/PipelineHealth/StepsFilter.tsx

@@ -1,3 +1,11 @@
+/**
+ * The steps filter determines which steps are shown on the step bar chart. It
+ * is a dropdown menu that allows the user to select, from the list of short
+ * pipeline step names, which steps to query. The filtering is applied client
+ * side, so the user can see the results of their selection immediately without
+ * needing to requery from the database.
+ */
+
 "use client";
 
 import { useRx } from "@effect-rx/rx-react";

components/PipelineHealth/Table.tsx

@@ -1,5 +1,10 @@
 "use client";
 
+import { useRxSuspenseSuccess, useRxValue } from "@effect-rx/rx-react";
+import { DateTime, HashSet } from "effect";
+import { ChevronDown } from "lucide-react";
+import Link from "next/link";
+
 import { steps2queryRx, tableDataRx } from "@/components/PipelineHealth/rx";
 import {
     DropdownMenu,
@@ -9,10 +14,6 @@ import {
 } from "@/components/ui/dropdown-menu";
 import { Table, TableBody, TableCaption, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
 import { splitLiteral } from "@/services/Domain";
-import { useRxSuspenseSuccess, useRxValue } from "@effect-rx/rx-react";
-import { DateTime, HashSet } from "effect";
-import { ChevronDown } from "lucide-react";
-import Link from "next/link";
 
 export const getSciURL = (filePath: string): string => {
     const start = filePath.indexOf("telescope_");