Docs: AudioNode + AnalyserNode

apps/common-app/src/examples/AudioVisualizer/AudioVisualizer.tsx

@@ -21,9 +21,7 @@ const AudioVisualizer: React.FC = () => {
   const [isPlaying, setIsPlaying] = useState(false);
   const [isLoading, setIsLoading] = useState(false);
 
-  const [times, setTimes] = useState<number[]>(
-    new Array(FFT_SIZE / 2).fill(127)
-  );
+  const [times, setTimes] = useState<number[]>(new Array(FFT_SIZE).fill(127));
   const [freqs, setFreqs] = useState<number[]>(new Array(FFT_SIZE / 2).fill(0));
 
   const audioContextRef = useRef<AudioContext | null>(null);
@@ -56,13 +54,14 @@ const AudioVisualizer: React.FC = () => {
       return;
     }
 
-    const bufferLength = analyserRef.current.frequencyBinCount;
+    const timesArrayLength = analyserRef.current.fftSize;
+    const frequencyArrayLength = analyserRef.current.frequencyBinCount;
 
-    const timesArray = new Array(bufferLength);
+    const timesArray = new Array(timesArrayLength);
     analyserRef.current.getByteTimeDomainData(timesArray);
     setTimes(timesArray);
 
-    const freqsArray = new Array(bufferLength);
+    const freqsArray = new Array(frequencyArrayLength);
     analyserRef.current.getByteFrequencyData(freqsArray);
     setFreqs(freqsArray);
 
@@ -107,6 +106,7 @@ const AudioVisualizer: React.FC = () => {
       <FreqTimeChart
         timeData={times}
         frequencyData={freqs}
+        fftSize={analyserRef.current?.fftSize || FFT_SIZE}
         frequencyBinCount={
           analyserRef.current?.frequencyBinCount || FFT_SIZE / 2
         }

apps/common-app/src/examples/AudioVisualizer/Charts.tsx

@@ -37,12 +37,13 @@ function weightWithIndex(value: number, index: number, indexMax: number) {
 
 interface ChartProps {
   data: number[];
+  fftSize: number;
   frequencyBinCount: number;
 }
 
 const TimeChart: React.FC<ChartProps> = (props) => {
   const { size } = useCanvas();
-  const { data, frequencyBinCount } = props;
+  const { data, fftSize } = props;
 
   const circlePaint = useMemo(() => {
     const paint = Skia.Paint();
@@ -61,12 +62,12 @@ const TimeChart: React.FC<ChartProps> = (props) => {
     const startHight = maxHeight - (maxHeight - 2 * INNER_RADIUS) / 2;
 
     return data.map((value, index) => {
-      const x = startWidth + (index * 2 * INNER_RADIUS) / frequencyBinCount;
-      const y = startHight - (value / 256) * 2 * INNER_RADIUS;
+      const x = startWidth + (index * 2 * INNER_RADIUS) / fftSize;
+      const y = startHight - (value / 255) * 2 * INNER_RADIUS;
 
       return vec(x, y);
     });
-  }, [size, data, frequencyBinCount]);
+  }, [size, data, fftSize]);
 
   return (
     <>

apps/common-app/src/examples/AudioVisualizer/FreqTimeChart.tsx

@@ -6,17 +6,23 @@ import Charts from './Charts';
 interface FreqTimeChartProps {
   timeData: number[];
   frequencyData: number[];
+  fftSize: number;
   frequencyBinCount: number;
 }
 
 const FreqTimeChart: React.FC<FreqTimeChartProps> = (props) => {
-  const { timeData, frequencyData, frequencyBinCount } = props;
+  const { timeData, frequencyData, fftSize, frequencyBinCount } = props;
 
   return (
     <>
-      <Charts.TimeChart data={timeData} frequencyBinCount={frequencyBinCount} />
+      <Charts.TimeChart
+        data={timeData}
+        fftSize={fftSize}
+        frequencyBinCount={frequencyBinCount}
+      />
       <Charts.FrequencyChart
         data={frequencyData}
+        fftSize={fftSize}
         frequencyBinCount={frequencyBinCount}
       />
     </>

packages/audiodocs/docs/nodes/_category_.json → packages/audiodocs/docs/core/_category_.json

@@ -1,5 +1,5 @@
 {
-  "label": "Nodes",
+  "label": "Core",
   "position": 3,
   "link": {
     "type": "generated-index"

packages/audiodocs/docs/core/audio-node.mdx

@@ -0,0 +1,54 @@
+---
+sidebar_position: 1
+---
+
+import { Optional, ReadOnly } from '@site/src/components/Badges';
+
+# AudioNode
+
+`AudioNode` interface serves as a versatile interface for constructing an audio processing graph, representing individual units of audio processing functionality.
+Each `AudioNode` is associated with a certain number of audio channels that facilitate the transfer of audio data through processing graph.
+
+We usually represent the channels with the standard abbreviations detailed in the table below:
+
+| Name | Number of channels | Channels |
+| :----: | :------: | :-------- |
+| Mono | 1 | 0: M - mono |
+| Stereo | 2 | 0: L - left <br /> 1: R - right |
+| Quad | 4 | 0: L - left <br /> 1: R - right <br /> 2: SL - surround left <br /> 3: SR - surround right |
+| Stereo | 6 | 0: L - left <br /> 1: R - right <br /> 2: C - center <br /> 3: LFE - subwoofer <br /> 4: SL - surround left <br /> 5: SR - surround right |
+
+## Read-only properties
+
+| Name | Type | Description |
+| :----: | :----: | :-------- |
+| `context` | [`BaseAudioContext`](/core/base-audio-context) | Returns the associated context. |
+| `numberOfInputs` | `number` | Returns value representing the number of input connections for the node. Source nodes are characterized by having a `numberOfInputs` value of 0. |
+| `numberOfOutputs` | `number` | Returns value representing the number of output connections for the node. Destination nodes are characterized by having a `numberOfOutputs` value of 0. |
+| `channelCount` | `number` | Returns value representing the number of channels utilized to decide how many channels are engaged during up-mixing or down-mixing with the node's inputs. |
+| `channelCountMode` | [`ChannelCountMode`](/types/channel-count-mode) | Returns an enumerated value that specifies the method by which channels are mixed between the node's inputs and outputs. |
+| `channelInterpretation` | [`ChannelInterpretation`](/types/channel-interpretation) | Returns an enumerated value that specifies how input channels are mapped to output channels when number of them is different. |
+
+## Methods
+
+### `connect`
+
+The `connect` method lets you connect one of the node's outputs to a destination.
+
+| Parameters | Type | Description |
+| :---: | :---: | :---- |
+| `destination` | [`AudioNode`](/core/audio-node) | Audio node to which to connect. |
+
+**Returns** `undefined`.
+
+### `disconnect`
+
+The `disconnect` method lets you disconnect one or more nodes from the node.
+
+| Parameters | Type | Description |
+| :---: | :---: | :---- |
+| `destination` <Optional /> | [`AudioNode`](/core/audio-node) | Audio node to which to connect. |
+
+If no arguments provided node disconnects from all outgoing connections.
+
+**Returns** `undefined`.

packages/audiodocs/docs/core/base-audio-context.mdx

@@ -0,0 +1,5 @@
+---
+sidebar_position: 2
+---
+
+# BaseAudioContext

packages/audiodocs/docs/other/_category_.json

@@ -1,6 +1,6 @@
 {
   "label": "Other",
-  "position": 4,
+  "position": 6,
   "link": {
     "type": "generated-index"
   }

packages/audiodocs/docs/types/_category_.json

@@ -0,0 +1,7 @@
+{
+  "label": "Types",
+  "position": 5,
+  "link": {
+    "type": "generated-index"
+  }
+}

packages/audiodocs/docs/types/channel-count-mode.mdx

@@ -0,0 +1,21 @@
+---
+sidebar_position: 1
+---
+
+# ChannelCountMode
+
+`ChannelCountMode` type determines how the number of input channels affects the number of output channels in an audio node.
+
+**Acceptable values:**
+- `max`
+
+  The number of channels is equal to the maximum number of channels of all connections. In this case, `channelCount` is ignored and only up-mixing happens.
+
+ - `clamped-max`
+
+  The number of channels is equal to the maximum number of channels of all connections, clamped to the value of `channelCount`(serves as the maximum permissible value).
+
+- `explicit`
+
+  The number of channels is defined by the value of `channelCount`.
+

packages/audiodocs/docs/types/channel-interpretation.mdx

@@ -0,0 +1,45 @@
+---
+sidebar_position: 2
+---
+
+# ChannelInterpretation
+
+`ChannelInterpretation` type specifies how input channels are mapped out to output channels when the number of them are different.
+
+**Acceptable values:**
+  - `speakers`
+
+  Use set of standard mapping rules for all combinations of common input and output setups.
+
+  - `discrete`
+
+  Covers all other cases. Mapping depends on relationship between number of input channels and number of output channels.
+
+
+## Channels mapping table
+
+### `speakers`
+
+| Number of input channels | Number of output channels | Mixing rules |
+| :------------------------: | :------------------------- | :------------ |
+| 1 (Mono) | 2 (Stereo) | output.L = input.M <br /> output.R = input.M |
+| 1 (Mono) | 4 (Quad) | output.L = input.M <br /> output.R = input.M <br /> output.SL = 0 <br /> output.SR = 0 |
+| 1 (Mono) | 6 (5.1) | output.L = 0 <br /> output.R = 0 <br /> output.C = input.M <br /> output.LFE = 0 <br /> output.SL = 0 <br /> output.SR = 0 |
+| 2 (Stereo) | 1 (Mono) | output.M = 0.5 \* (input.L + input.R) |
+| 2 (Stereo) | 4 (Quad) | output.L = input.L <br /> output.R = input.R <br /> output.SL = 0 <br /> output.SR = 0 |
+| 2 (Stereo) | 6 (5.1) | output.L = input.L <br /> output.R = input.R <br /> output.C = 0 <br /> output.LFE = 0 <br /> output.SL = 0 <br /> output.SR = 0 |
+| 4 (Quad) | 1 (Mono) | output.M = 0.25 \* (input.L + input.R + input.SL + input.SR) |
+| 4 (Quad) | 2 (Stereo) | output.L = 0.5 \* (input.L + input.SL) <br /> output.R = 0.5 \* (input.R + input.SR) |
+| 4 (Quad) | 6 (5.1) | output.L = input.L <br /> output.R = input.R <br /> output.C = 0 <br /> output.LFE = 0 <br /> output.SL = input.SL <br /> output.SR = input.SR |
+| 6 (5.1) | 1 (Mono) | output.M = 0.7071 \* (input.L + input.R) + input.C <br /> + 0.5 \* (input.SL + input.SR) |
+| 6 (5.1) | 2 (Stereo) | output.L = input.L + 0.7071 \* (input.C + input.SL) <br /> output.R = input.R + 0.7071 \* (input.C + input.SR) |
+| 6 (5.1) | 4 (Quad) | output.L = input.L + 0.7071 \* input.C <br /> output.R = input.R + 0.7071 \* input.C <br /> output.SL = input.SL <br /> output.SR = input.SR |
+
+### `discrete`
+
+| Number of input channels | Number of output channels | Mixing rules |
+| :------------------------: | :------------------------- | :------------ |
+| x | y where y > x | Fill each output channel with its counterpart(channel with same number), rest of output channels are silent channels |
+| x | y where y < x | Fill each output channel with its counterpart(channel with same number), rest of input channels are skipped |
+
+

packages/audiodocs/docs/visualization/_category_.json

@@ -0,0 +1,7 @@
+{
+  "label": "Visualization",
+  "position": 4,
+  "link": {
+    "type": "generated-index"
+  }
+}

packages/audiodocs/docs/visualization/analyser-node.mdx

@@ -0,0 +1,103 @@
+---
+sidebar_position: 1
+---
+
+import AudioNodePropsTable from "@site/src/components/AudioNodePropsTable"
+
+# AnalyserNode
+
+`AnalyserNode` interface represents a node providing two core functionalities: extracting time-domain data and frequency-domain data from audio signals.
+It is an [`AudioNode`](/core/audio-node) that passes the audio data unchanged from input to output, but allows you to take passed data and process it.
+
+#### [`AudioNode`](/core/audio-node#read-only-properties) properties
+
+<AudioNodePropsTable numberOfInputs={1} numberOfOutputs={1} channelCount={2} channelCountMode={"max"} channelInterpretation={"speakers"} />
+
+#### Time domain vs Frequency domain
+
+![time-domain-vs-frequency-domain](/img/time_domain_vs_frequency_domain.jpg)
+
+A time-domain graph illustrates how a signal evolves over time, displaying changes in amplitude or intensity as time progresses.
+In contrast, a frequency-domain graph reveals how the signal's energy or power is distributed across different frequency bands, highlighting the presence and strength of various frequency components over a specified range.
+
+## Properties
+
+| Name | Type | Description |
+| :----: | :----: | :-------- |
+| `fftSize` | `number` | Returns integer value representing size of [Fast Fourier Transform](https://en.wikipedia.org/wiki/Fast_Fourier_transform) used to determine frequency domain. In general it is size of returning time-domain data. |
+| `minDecibels` | `number` | Returns float value representing the minimum value for the range of results from [`getByteFrequencyData()`](/visualization/analyser-node#getbytefrequencydata). |
+| `maxDecibels` | `number` | Returns float value representing the maximum value for the range of results from [`getByteFrequencyData()`](/visualization/analyser-node#getbytefrequencydata). |
+| `smoothingTimeConstant` | `number` | Returns float value representing averaging constant with the last analysis frame. In general the higher value the smoother is the transition between values over time. |
+
+## Read-only properties
+
+| Name | Type | Description |
+| :----: | :----: | :-------- |
+| `frequencyBinCount` | `number` | Returns integer value representing amount of the data obtained in frequency domain, half of the `fftSize` property. |
+
+## Methods
+
+### `getFloatFrequencyData`
+
+The `getFloatFrequencyData` method copies current frequency data into given array.
+Each value in the array represents the decibel value for a specific frequency.
+
+| Parameters | Type | Description |
+| :---: | :---: | :---- |
+| `array` | `number[]` | The array to which frequency data will be copied. |
+
+**Returns** `undefined`.
+
+### `getByteFrequencyData`
+
+The `getByteFrequencyData` method copies current frequency data into given array.
+Each value in the array is within the range 0 to 255.
+
+| Parameters | Type | Description |
+| :---: | :---: | :---- |
+| `array` | `number[]` | The array to which frequency data will be copied. |
+
+**Returns** `undefined`.
+
+### `getFloatTimeDomainData`
+
+The `getFloatTimeDomainData` method copies current time-domain data into given array.
+Each value in the array is the magnitude of the signal at a particular time.
+
+| Parameters | Type | Description |
+| :---: | :---: | :---- |
+| `array` | `number[]` | The array to which time-domain data will be copied. |
+
+**Returns** `undefined`.
+
+### `getByteTimeDomainData`
+
+The `getByteTimeDomainData` method copies current time-domain data into given array.
+Each value in the array is within the range 0 to 255, where value of 127 indicates silence.
+
+| Parameters | Type | Description |
+| :---: | :---: | :---- |
+| `array` | `number[]` | The array to which time-domain data will be copied. |
+
+**Returns** `undefined`.
+
+## Remarks
+
+#### `fftSize`
+- Default value is 2048.
+- Must be a power of 2 between 32 and 32768.
+
+#### `minDecibels`
+- Default value is -100 dB.
+- 0 dB([decibel](https://en.wikipedia.org/wiki/Decibel)) is the loudest possible sound, -10 dB is a 10th of that.
+- When getting data from [`getByteFrequencyData()`](/visualization/analyser-node#getbytefrequencydata), any frequency with amplitude lower then `minDecibels` will be returned as 0.
+
+#### `maxDecibels`
+- Default value is -30 dB.
+- 0 dB([decibel](https://en.wikipedia.org/wiki/Decibel)) is the loudest possible sound, -10 dB is a 10th of that.
+- When getting data from [`getByteFrequencyData()`](/visualization/analyser-node#getbytefrequencydata), any frequency with amplitude higher then `maxDecibels` will be returned as 255.
+
+#### `smoothingTimeConstant`
+- Default value is 0.8.
+- From range 0 to 1.
+- 0 means no averaging, 1 means "overlap the previous and current buffer quite a lot while computing the value".