"Fossies" - the Fresh Open Source Software Archive

Member "jitsi-meet-7561/react/features/stream-effects/noise-suppression/NoiseSuppressorWorklet.ts" (29 Sep 2023, 8016 Bytes) of package /linux/misc/jitsi-meet-7561.tar.gz:

    1 // @ts-expect-error
    2 import { createRNNWasmModuleSync } from '@jitsi/rnnoise-wasm';
    3 
    4 import { leastCommonMultiple } from '../../base/util/math';
    5 import RnnoiseProcessor from '../rnnoise/RnnoiseProcessor';
    6 
    7 
    8 /**
    9  * Audio worklet which will denoise targeted audio stream using rnnoise.
   10  */
   11 class NoiseSuppressorWorklet extends AudioWorkletProcessor {
   12     /**
   13      * RnnoiseProcessor instance.
   14      */
   15     private _denoiseProcessor: RnnoiseProcessor;
   16 
   17     /**
   18      * Audio worklets work with a predefined sample rate of 128.
   19      */
   20     private _procNodeSampleRate = 128;
   21 
   22     /**
   23      * PCM Sample size expected by the denoise processor.
   24      */
   25     private _denoiseSampleSize: number;
   26 
   27     /**
   28      * Circular buffer data used for efficient memory operations.
   29      */
   30     private _circularBufferLength: number;
   31 
   32     private _circularBuffer: Float32Array;
   33 
   34     /**
   35      * The circular buffer uses a couple of indexes to track data segments. Input data from the stream is
   36      * copied to the circular buffer as it comes in, one `procNodeSampleRate` sized sample at a time.
   37      * _inputBufferLength denotes the current length of all gathered raw audio segments.
   38      */
   39     private _inputBufferLength = 0;
   40 
   41     /**
   42      * Denoising is done directly on the circular buffer using subArray views, but because
   43      * `procNodeSampleRate` and `_denoiseSampleSize` have different sizes, denoised samples lag behind
   44      * the current gathered raw audio samples so we need a different index, `_denoisedBufferLength`.
   45      */
   46     private _denoisedBufferLength = 0;
   47 
   48     /**
   49      * Once enough data has been denoised (size of procNodeSampleRate) it's sent to the
   50      * output buffer, `_denoisedBufferIndx` indicates the start index on the circular buffer
   51      * of denoised data not yet sent.
   52      */
   53     private _denoisedBufferIndx = 0;
   54 
   55     /**
   56      * C'tor.
   57      */
   58     constructor() {
   59         super();
   60 
   61         /**
   62          * The wasm module needs to be compiled to load synchronously as the audio worklet `addModule()`
   63          * initialization process does not wait for the resolution of promises in the AudioWorkletGlobalScope.
   64          */
   65         this._denoiseProcessor = new RnnoiseProcessor(createRNNWasmModuleSync());
   66 
   67         /**
   68          * PCM Sample size expected by the denoise processor.
   69          */
   70         this._denoiseSampleSize = this._denoiseProcessor.getSampleLength();
   71 
   72         /**
   73          * In order to avoid unnecessary memory related operations a circular buffer was used.
   74          * Because the audio worklet input array does not match the sample size required by rnnoise two cases can occur
   75          * 1. There is not enough data in which case we buffer it.
   76          * 2. There is enough data but some residue remains after the call to `processAudioFrame`, so its buffered
   77          * for the next call.
   78          * A problem arises when the circular buffer reaches the end and a rollover is required, namely
   79          * the residue could potentially be split between the end of buffer and the beginning and would
   80          * require some complicated logic to handle. Using the lcm as the size of the buffer will
   81          * guarantee that by the time the buffer reaches the end the residue will be a multiple of the
   82          * `procNodeSampleRate` and the residue won't be split.
   83          */
   84         this._circularBufferLength = leastCommonMultiple(this._procNodeSampleRate, this._denoiseSampleSize);
   85         this._circularBuffer = new Float32Array(this._circularBufferLength);
   86     }
   87 
   88     /**
   89      * Worklet interface process method. The inputs parameter contains PCM audio that is then sent to rnnoise.
   90      * Rnnoise only accepts PCM samples of 480 bytes whereas `process` handles 128 sized samples, we take this into
   91      * account using a circular buffer.
   92      *
   93      * @param {Float32Array[]} inputs - Array of inputs connected to the node, each of them with their associated
   94      * array of channels. Each channel is an array of 128 pcm samples.
   95      * @param {Float32Array[]} outputs - Array of outputs similar to the inputs parameter structure, expected to be
   96      * filled during the execution of `process`. By default each channel is zero filled.
   97      * @returns {boolean} - Boolean value that returns whether or not the processor should remain active. Returning
   98      * false will terminate it.
   99      */
  100     process(inputs: Float32Array[][], outputs: Float32Array[][]) {
  101 
  102         // We expect the incoming track to be mono, if a stereo track is passed only on of its channels will get
  103         // denoised and sent pack.
  104         // TODO Technically we can denoise both channel however this might require a new rnnoise context, some more
  105         // investigation is required.
  106         const inData = inputs[0][0];
  107         const outData = outputs[0][0];
  108 
  109         // Exit out early if there is no input data (input node not connected/disconnected)
  110         // as rest of worklet will crash otherwise
  111         if (!inData) {
  112             return true;
  113         }
  114 
  115         // Append new raw PCM sample.
  116         this._circularBuffer.set(inData, this._inputBufferLength);
  117         this._inputBufferLength += inData.length;
  118 
  119         // New raw samples were just added, start denoising frames, _denoisedBufferLength gives us
  120         // the position at which the previous denoise iteration ended, basically it takes into account
  121         // residue data.
  122         for (; this._denoisedBufferLength + this._denoiseSampleSize <= this._inputBufferLength;
  123             this._denoisedBufferLength += this._denoiseSampleSize) {
  124             // Create view of circular buffer so it can be modified in place, removing the need for
  125             // extra copies.
  126 
  127             const denoiseFrame = this._circularBuffer.subarray(
  128                 this._denoisedBufferLength,
  129                 this._denoisedBufferLength + this._denoiseSampleSize
  130             );
  131 
  132             this._denoiseProcessor.processAudioFrame(denoiseFrame, true);
  133         }
  134 
  135         // Determine how much denoised audio is available, if the start index of denoised samples is smaller
  136         // then _denoisedBufferLength that means a rollover occurred.
  137         let unsentDenoisedDataLength;
  138 
  139         if (this._denoisedBufferIndx > this._denoisedBufferLength) {
  140             unsentDenoisedDataLength = this._circularBufferLength - this._denoisedBufferIndx;
  141         } else {
  142             unsentDenoisedDataLength = this._denoisedBufferLength - this._denoisedBufferIndx;
  143         }
  144 
  145         // Only copy denoised data to output when there's enough of it to fit the exact buffer length.
  146         // e.g. if the buffer size is 1024 samples but we only denoised 960 (this happens on the first iteration)
  147         // nothing happens, then on the next iteration 1920 samples will be denoised so we send 1024 which leaves
  148         // 896 for the next iteration and so on.
  149         if (unsentDenoisedDataLength >= outData.length) {
  150             const denoisedFrame = this._circularBuffer.subarray(
  151                 this._denoisedBufferIndx,
  152                 this._denoisedBufferIndx + outData.length
  153             );
  154 
  155             outData.set(denoisedFrame, 0);
  156             this._denoisedBufferIndx += outData.length;
  157         }
  158 
  159         // When the end of the circular buffer has been reached, start from the beginning. By the time the index
  160         // starts over, the data from the begging is stale (has already been processed) and can be safely
  161         // overwritten.
  162         if (this._denoisedBufferIndx === this._circularBufferLength) {
  163             this._denoisedBufferIndx = 0;
  164         }
  165 
  166         // Because the circular buffer's length is the lcm of both input size and the processor's sample size,
  167         // by the time we reach the end with the input index the denoise length index will be there as well.
  168         if (this._inputBufferLength === this._circularBufferLength) {
  169             this._inputBufferLength = 0;
  170             this._denoisedBufferLength = 0;
  171         }
  172 
  173         return true;
  174     }
  175 }
  176 
  177 registerProcessor('NoiseSuppressorWorklet', NoiseSuppressorWorklet);