Realtime Audio Analyzer Demo

1import { AudioDataProvider } from "./AudioDataProvider";
2import { Radix2FFT } from "./Radix2FFT";
3import { appTheme } from "../../../theme";
4import {
5    XyDataSeries,
6    UniformHeatmapDataSeries,
7    TextAnnotation,
8    ECoordinateMode,
9    EHorizontalAnchorPoint,
10    EVerticalAnchorPoint,
11    SciChartSurface,
12    NumericAxis,
13    EAutoRange,
14    NumberRange,
15    FastLineRenderableSeries,
16    LogarithmicAxis,
17    ENumericFormat,
18    EAxisAlignment,
19    FastMountainRenderableSeries,
20    EllipsePointMarker,
21    PaletteFactory,
22    GradientParams,
23    Point,
24    UniformHeatmapRenderableSeries,
25    HeatmapColorMap,
26} from "scichart";
27
28const AUDIO_STREAM_BUFFER_SIZE = 2048;
29
30export const getChartsInitializationApi = () => {
31    const dataProvider = new AudioDataProvider();
32
33    const bufferSize = dataProvider.bufferSize;
34    const sampleRate = dataProvider.sampleRate;
35
36    const fft = new Radix2FFT(bufferSize);
37
38    const hzPerDataPoint = sampleRate / bufferSize;
39    const fftSize = fft.fftSize;
40    const fftCount = 200;
41
42    let fftXValues: number[];
43    let spectrogramValues: number[][];
44
45    let audioDS: XyDataSeries;
46    let historyDS: XyDataSeries;
47    let fftDS: XyDataSeries;
48    let spectrogramDS: UniformHeatmapDataSeries;
49
50    let hasAudio: boolean;
51
52    const helpText = new TextAnnotation({
53        x1: 0,
54        y1: 0,
55        xAxisId: "history",
56        xCoordinateMode: ECoordinateMode.Relative,
57        yCoordinateMode: ECoordinateMode.Relative,
58        horizontalAnchorPoint: EHorizontalAnchorPoint.Left,
59        verticalAnchorPoint: EVerticalAnchorPoint.Top,
60        text: "This example requires microphone permissions.  Please click Allow in the popup.",
61        textColor: "#FFFFFF88",
62    });
63
64    function updateAnalysers(frame: number): void {
65        // Make sure Audio is initialized
66        if (dataProvider.initialized === false) {
67            return;
68        }
69
70        // Get audio data
71        const audioData = dataProvider.next();
72
73        // Update Audio Chart. When fifoCapacity is set, data automatically scrolls
74        audioDS.appendRange(audioData.xData, audioData.yData);
75
76        // Update History. When fifoCapacity is set, data automatically scrolls
77        historyDS.appendRange(audioData.xData, audioData.yData);
78
79        // Perform FFT
80        const fftData = fft.run(audioData.yData);
81
82        // Update FFT Chart. Clear() and appendRange() is a fast replace for data (if same size)
83        fftDS.clear();
84        fftDS.appendRange(fftXValues, fftData);
85
86        // Update Spectrogram Chart
87        spectrogramValues.shift();
88        spectrogramValues.push(fftData);
89        spectrogramDS.setZValues(spectrogramValues);
90    }
91
92    // AUDIO CHART
93    const initAudioChart = async (rootElement: string | HTMLDivElement) => {
94        // Create a chart for the audio
95        const { sciChartSurface, wasmContext } = await SciChartSurface.create(rootElement, {
96            theme: appTheme.SciChartJsTheme,
97        });
98
99        // Create an XAxis for the live audio
100        const xAxis = new NumericAxis(wasmContext, {
101            id: "audio",
102            autoRange: EAutoRange.Always,
103            drawLabels: false,
104            drawMinorTickLines: false,
105            drawMajorTickLines: false,
106            drawMajorBands: false,
107            drawMinorGridLines: false,
108            drawMajorGridLines: false,
109        });
110        sciChartSurface.xAxes.add(xAxis);
111
112        // Create an XAxis for the history of the audio on the same chart
113        const xhistAxis = new NumericAxis(wasmContext, {
114            id: "history",
115            autoRange: EAutoRange.Always,
116            drawLabels: false,
117            drawMinorGridLines: false,
118            drawMajorTickLines: false,
119        });
120        sciChartSurface.xAxes.add(xhistAxis);
121
122        // Create a YAxis for the audio data
123        const yAxis = new NumericAxis(wasmContext, {
124            autoRange: EAutoRange.Never,
125            visibleRange: new NumberRange(-32768 * 0.8, 32767 * 0.8), // [short.MIN. short.MAX]
126            drawLabels: false,
127            drawMinorTickLines: false,
128            drawMajorTickLines: false,
129            drawMajorBands: false,
130            drawMinorGridLines: false,
131            drawMajorGridLines: false,
132        });
133        sciChartSurface.yAxes.add(yAxis);
134
135        // Initializing a series with fifoCapacity enables scrolling behaviour and auto discarding old data
136        audioDS = new XyDataSeries(wasmContext, { fifoCapacity: AUDIO_STREAM_BUFFER_SIZE });
137
138        // Fill the data series with zero values
139        for (let i = 0; i < AUDIO_STREAM_BUFFER_SIZE; i++) {
140            audioDS.append(0, 0);
141        }
142
143        // Add a line series for the live audio data
144        // using XAxisId=audio for the live audio trace scaling
145        const rs = new FastLineRenderableSeries(wasmContext, {
146            xAxisId: "audio",
147            stroke: "#4FBEE6",
148            strokeThickness: 2,
149            dataSeries: audioDS,
150        });
151
152        sciChartSurface.renderableSeries.add(rs);
153
154        // Initializing a series with fifoCapacity enables scrolling behaviour and auto discarding old data.
155        historyDS = new XyDataSeries(wasmContext, { fifoCapacity: AUDIO_STREAM_BUFFER_SIZE * fftCount });
156        for (let i = 0; i < AUDIO_STREAM_BUFFER_SIZE * fftCount; i++) {
157            historyDS.append(0, 0);
158        }
159
160        // Add a line series for the historical audio data
161        // using the XAxisId=history for separate scaling for this trace
162        const histrs = new FastLineRenderableSeries(wasmContext, {
163            stroke: "#208EAD33",
164            strokeThickness: 1,
165            opacity: 0.5,
166            xAxisId: "history",
167            dataSeries: historyDS,
168        });
169        sciChartSurface.renderableSeries.add(histrs);
170
171        // Add instructions
172        sciChartSurface.annotations.add(helpText);
173
174        hasAudio = await dataProvider.initAudio();
175
176        return { sciChartSurface };
177    };
178
179    // FFT CHART
180    const initFftChart = async (rootElement: string | HTMLDivElement) => {
181        const { sciChartSurface, wasmContext } = await SciChartSurface.create(rootElement, {
182            theme: appTheme.SciChartJsTheme,
183        });
184        const xAxis = new LogarithmicAxis(wasmContext, {
185            logBase: 10,
186            labelFormat: ENumericFormat.SignificantFigures,
187            maxAutoTicks: 5,
188            axisTitleStyle: { fontSize: 10 },
189            drawMinorGridLines: false,
190            drawMinorTickLines: false,
191            drawMajorTickLines: false,
192        });
193        sciChartSurface.xAxes.add(xAxis);
194
195        const yAxis = new NumericAxis(wasmContext, {
196            axisAlignment: EAxisAlignment.Left,
197            visibleRange: new NumberRange(0, 80),
198            growBy: new NumberRange(0.1, 0.1),
199            drawMinorGridLines: false,
200            drawMinorTickLines: false,
201            drawMajorTickLines: false,
202            labelPrecision: 0,
203            axisTitleStyle: { fontSize: 10 },
204        });
205        sciChartSurface.yAxes.add(yAxis);
206
207        fftDS = new XyDataSeries(wasmContext);
208        fftXValues = new Array<number>(fftSize);
209        for (let i = 0; i < fftSize; i++) {
210            fftXValues[i] = (i + 1) * hzPerDataPoint;
211        }
212
213        // Make a column chart with a gradient palette on the stroke only
214        const rs = new FastMountainRenderableSeries(wasmContext, {
215            dataSeries: fftDS,
216            pointMarker: new EllipsePointMarker(wasmContext, { width: 9, height: 9 }),
217            strokeThickness: 3,
218            paletteProvider: PaletteFactory.createGradient(
219                wasmContext,
220                new GradientParams(new Point(0, 0), new Point(1, 1), [
221                    { offset: 0, color: "#36B8E6" },
222                    { offset: 0.001, color: "#5D8CC2" },
223                    { offset: 0.01, color: "#8166A2" },
224                    { offset: 0.1, color: "#AE418C" },
225                    { offset: 1.0, color: "#CA5B79" },
226                ]),
227                {
228                    enableStroke: true,
229                    enableFill: true,
230                    enablePointMarkers: true,
231                    fillOpacity: 0.17,
232                    pointMarkerOpacity: 0.37,
233                }
234            ),
235        });
236        sciChartSurface.renderableSeries.add(rs);
237
238        return { sciChartSurface };
239    };
240
241    // SPECTROGRAM CHART
242    const initSpectogramChart = async (rootElement: string | HTMLDivElement) => {
243        spectrogramValues = new Array<number[]>(fftCount);
244        for (let i = 0; i < fftCount; i++) {
245            spectrogramValues[i] = new Array<number>(fftSize);
246            for (let j = 0; j < fftSize; j++) {
247                spectrogramValues[i][j] = 0;
248            }
249        }
250
251        const { sciChartSurface, wasmContext } = await SciChartSurface.create(rootElement, {
252            theme: appTheme.SciChartJsTheme,
253        });
254
255        const xAxis = new NumericAxis(wasmContext, {
256            autoRange: EAutoRange.Always,
257            drawLabels: false,
258            drawMinorTickLines: false,
259            drawMajorTickLines: false,
260        });
261        sciChartSurface.xAxes.add(xAxis);
262
263        const yAxis = new NumericAxis(wasmContext, {
264            autoRange: EAutoRange.Always,
265            drawLabels: false,
266            drawMinorTickLines: false,
267            drawMajorTickLines: false,
268        });
269        sciChartSurface.yAxes.add(yAxis);
270
271        spectrogramDS = new UniformHeatmapDataSeries(wasmContext, {
272            xStart: 0,
273            xStep: 1,
274            yStart: 0,
275            yStep: 1,
276            zValues: spectrogramValues,
277        });
278
279        const rs = new UniformHeatmapRenderableSeries(wasmContext, {
280            dataSeries: spectrogramDS,
281            colorMap: new HeatmapColorMap({
282                minimum: 0,
283                maximum: 70,
284                gradientStops: [
285                    { offset: 0, color: "#000000" },
286                    { offset: 0.25, color: "#800080" },
287                    { offset: 0.5, color: "#FF0000" },
288                    { offset: 0.75, color: "#FFFF00" },
289                    { offset: 1, color: "#FFFFFF" },
290                ],
291            }),
292        });
293        sciChartSurface.renderableSeries.add(rs);
294
295        return { sciChartSurface };
296    };
297
298    const onAllChartsInit = () => {
299        if (!hasAudio) {
300            console.log("dataProvider", dataProvider);
301            if (dataProvider.permissionError) {
302                helpText.text =
303                    "We were not able to access your microphone.  This may be because you did not accept the permissions.  Open your browser security settings and remove the block on microphone permissions from this site, then reload the page.";
304            } else if (!window.isSecureContext) {
305                helpText.text = "Cannot get microphone access if the site is not localhost or on https";
306            } else {
307                helpText.text = "There was an error trying to get microphone access.  Check the console";
308            }
309
310            return { startUpdate: () => {}, stopUpdate: () => {}, cleanup: () => {} };
311        } else {
312            helpText.text = "This example uses your microphone to generate waveforms. Say something!";
313
314            // START ANIMATION
315
316            let frameCounter = 0;
317            const updateChart = () => {
318                if (!dataProvider.isDeleted) {
319                    updateAnalysers(frameCounter++);
320                }
321            };
322
323            let timerId: NodeJS.Timeout;
324
325            const startUpdate = () => {
326                timerId = setInterval(updateChart, 20);
327            };
328
329            const stopUpdate = () => {
330                clearInterval(timerId);
331            };
332
333            const cleanup = () => {
334                dataProvider.closeAudio();
335            };
336
337            return { startUpdate, stopUpdate, cleanup };
338        }
339    };
340
341    return { initAudioChart, initFftChart, initSpectogramChart, onAllChartsInit };
342};
343