"Fossies" - the Fresh Open Source Software Archive

Member "jitsi-meet-7555/react/features/stream-effects/rnnoise/RnnoiseProcessor.ts" (28 Sep 2023, 6375 Bytes) of package /linux/misc/jitsi-meet-7555.tar.gz:

    1 /* eslint-disable no-bitwise */
    2 
    3 interface IRnnoiseModule extends EmscriptenModule {
    4     _rnnoise_create: () => number;
    5     _rnnoise_destroy: (context: number) => void;
    6     _rnnoise_process_frame: (context: number, input: number, output: number) => number;
    7 }
    8 
    9 /**
   10  * Constant. Rnnoise default sample size, samples of different size won't work.
   11  */
   12 export const RNNOISE_SAMPLE_LENGTH = 480;
   13 
   14 /**
   15  *  Constant. Rnnoise only takes inputs of 480 PCM float32 samples thus 480*4.
   16  */
   17 const RNNOISE_BUFFER_SIZE: number = RNNOISE_SAMPLE_LENGTH * 4;
   18 
   19 /**
   20  *  Constant. Rnnoise only takes operates on 44.1Khz float 32 little endian PCM.
   21  */
   22 const PCM_FREQUENCY = 44100;
   23 
   24 /**
   25  * Used to shift a 32 bit number by 16 bits.
   26  */
   27 const SHIFT_16_BIT_NR = 32768;
   28 
   29 /**
   30  * Represents an adaptor for the rnnoise library compiled to webassembly. The class takes care of webassembly
   31  * memory management and exposes rnnoise functionality such as PCM audio denoising and VAD (voice activity
   32  * detection) scores.
   33  */
   34 export default class RnnoiseProcessor {
   35     /**
   36      * Rnnoise context object needed to perform the audio processing.
   37      */
   38     private _context: number;
   39 
   40     /**
   41      * State flag, check if the instance was destroyed.
   42      */
   43     private _destroyed = false;
   44 
   45     /**
   46      * WASM interface through which calls to rnnoise are made.
   47      */
   48     private _wasmInterface: IRnnoiseModule;
   49 
   50     /**
   51      * WASM dynamic memory buffer used as input for rnnoise processing method.
   52      */
   53     private _wasmPcmInput: number;
   54 
   55     /**
   56      * The Float32Array index representing the start point in the wasm heap of the _wasmPcmInput buffer.
   57      */
   58     private _wasmPcmInputF32Index: number;
   59 
   60     /**
   61      * Constructor.
   62      *
   63      * @class
   64      * @param {Object} wasmInterface - WebAssembly module interface that exposes rnnoise functionality.
   65      */
   66     constructor(wasmInterface: IRnnoiseModule) {
   67         // Considering that we deal with dynamic allocated memory employ exception safety strong guarantee
   68         // i.e. in case of exception there are no side effects.
   69         try {
   70             this._wasmInterface = wasmInterface;
   71 
   72             // For VAD score purposes only allocate the buffers once and reuse them
   73             this._wasmPcmInput = this._wasmInterface._malloc(RNNOISE_BUFFER_SIZE);
   74 
   75             this._wasmPcmInputF32Index = this._wasmPcmInput >> 2;
   76 
   77             if (!this._wasmPcmInput) {
   78                 throw Error('Failed to create wasm input memory buffer!');
   79             }
   80 
   81             this._context = this._wasmInterface._rnnoise_create();
   82         } catch (error) {
   83             // release can be called even if not all the components were initialized.
   84             this.destroy();
   85             throw error;
   86         }
   87     }
   88 
   89     /**
   90      * Release resources associated with the wasm context. If something goes downhill here
   91      * i.e. Exception is thrown, there is nothing much we can do.
   92      *
   93      * @returns {void}
   94      */
   95     _releaseWasmResources(): void {
   96         // For VAD score purposes only allocate the buffers once and reuse them
   97         if (this._wasmPcmInput) {
   98             this._wasmInterface._free(this._wasmPcmInput);
   99         }
  100 
  101         if (this._context) {
  102             this._wasmInterface._rnnoise_destroy(this._context);
  103         }
  104     }
  105 
  106     /**
  107      * Rnnoise can only operate on a certain PCM array size.
  108      *
  109      * @returns {number} - The PCM sample array size as required by rnnoise.
  110      */
  111     getSampleLength(): number {
  112         return RNNOISE_SAMPLE_LENGTH;
  113     }
  114 
  115     /**
  116      * Rnnoise can only operate on a certain format of PCM sample namely float 32 44.1Kz.
  117      *
  118      * @returns {number} - PCM sample frequency as required by rnnoise.
  119      */
  120     getRequiredPCMFrequency(): number {
  121         return PCM_FREQUENCY;
  122     }
  123 
  124     /**
  125      * Release any resources required by the rnnoise context this needs to be called
  126      * before destroying any context that uses the processor.
  127      *
  128      * @returns {void}
  129      */
  130     destroy(): void {
  131         // Attempting to release a non initialized processor, do nothing.
  132         if (this._destroyed) {
  133             return;
  134         }
  135 
  136         this._releaseWasmResources();
  137 
  138         this._destroyed = true;
  139     }
  140 
  141     /**
  142      * Calculate the Voice Activity Detection for a raw Float32 PCM sample Array.
  143      * The size of the array must be of exactly 480 samples, this constraint comes from the rnnoise library.
  144      *
  145      * @param {Float32Array} pcmFrame - Array containing 32 bit PCM samples.
  146      * @returns {Float} Contains VAD score in the interval 0 - 1 i.e. 0.90.
  147      */
  148     calculateAudioFrameVAD(pcmFrame: Float32Array): number {
  149         return this.processAudioFrame(pcmFrame);
  150     }
  151 
  152     /**
  153      * Process an audio frame, optionally denoising the input pcmFrame and returning the Voice Activity Detection score
  154      * for a raw Float32 PCM sample Array.
  155      * The size of the array must be of exactly 480 samples, this constraint comes from the rnnoise library.
  156      *
  157      * @param {Float32Array} pcmFrame - Array containing 32 bit PCM samples. Parameter is also used as output
  158      * when {@code shouldDenoise} is true.
  159      * @param {boolean} shouldDenoise - Should the denoised frame be returned in pcmFrame.
  160      * @returns {Float} Contains VAD score in the interval 0 - 1 i.e. 0.90 .
  161      */
  162     processAudioFrame(pcmFrame: Float32Array, shouldDenoise: Boolean = false): number {
  163         // Convert 32 bit Float PCM samples to 16 bit Float PCM samples as that's what rnnoise accepts as input
  164         for (let i = 0; i < RNNOISE_SAMPLE_LENGTH; i++) {
  165             this._wasmInterface.HEAPF32[this._wasmPcmInputF32Index + i] = pcmFrame[i] * SHIFT_16_BIT_NR;
  166         }
  167 
  168         // Use the same buffer for input/output, rnnoise supports this behavior
  169         const vadScore = this._wasmInterface._rnnoise_process_frame(
  170             this._context,
  171             this._wasmPcmInput,
  172             this._wasmPcmInput
  173         );
  174 
  175         // Rnnoise denoises the frame by default but we can avoid unnecessary operations if the calling
  176         // client doesn't use the denoised frame.
  177         if (shouldDenoise) {
  178             // Convert back to 32 bit PCM
  179             for (let i = 0; i < RNNOISE_SAMPLE_LENGTH; i++) {
  180                 pcmFrame[i] = this._wasmInterface.HEAPF32[this._wasmPcmInputF32Index + i] / SHIFT_16_BIT_NR;
  181             }
  182         }
  183 
  184         return vadScore;
  185     }
  186 }