Haikang SDK/Ehome protocol/RTSP protocol/GB28181 security video cloud service EasyCVR front-end audio collection process introduction

Haikang SDK/Ehome protocol/RTSP protocol/GB28181 security video cloud service EasyCVR can be cascaded through the GB28181 protocol. If the camera or device supports audio, EasyCVR can also perform audio capture.

The front-end js of EasyCVR video platform uses webapi to collect device audio. Pay special attention to getUserMedia in the case of non-localhost and 127, and https needs to be turned on.

Basic steps

1. Use webrtc's getUserMedia method to get device audio input, and use audioprocess to get audio stream (pcm stream, range -1 to 1).

2. Resampling, the front-end sampling rate is 48000, the back-end needs sampling rate is 8000, all need to be combined and compressed

3. Value conversion. The input data obtained in each chunk is a Float32Array fixed array with a length of 4096, which means that each sampling point information is stored in 32-bit floating point.

The 32-bit stored sampling frame value uses -1 to 1 to map the 16-bit storage range -32768~32767.

The following is the conversion code:

function floatTo16BitPCM(output, offset, input) {
for (let i = 0; i < input.length; i++, offset += 2) {
//下面一行代码保证了采样帧的值在-1到1之间，因为有可能在多声道合并或其他状况下超出范围 

        let s = Math.max(-1, Math.min(1, input[i]));
//将32位浮点映射为16位整形表示的值
        output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
    }
}

Description:

If s>0 actually maps 01 to 032767, and the first sign bit of a positive number is 0, so 32767 corresponds to 0111 1111 1111 1111, which is 0x7FFF, just multiply s as a coefficient; when s is For negative numbers, you need to map 0-1 to 0-32768, so the value of s can also be directly used as a scale factor for conversion calculations. When negative numbers are stored in the memory, you need to use the complement code. The complement code is the original code in addition to the sign bit. The bit is inverted and then +1 is obtained, so the original code of -32768 is 1000 0000 0000 0000 (the overflowed bit is directly discarded), except for the sign bit, the bit is inverted to get 1111 1111 1111 1111, and finally +1 is calculated to get 1000 0000 0000 0000 (overflowed bits are also directly discarded), expressed in hexadecimal is 0x8000. By the way, the complement exists to make the addition of positive and negative values equal to 0 in binary form.

4. Websocket establishes a client link to send data, and observes that the collection callback is triggered once about 80ms, and the data is sent in the triggered callback function

<!DOCTYPE html>
<html>
    <head>
        <meta charset="UTF-8">
        <meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
        <meta name="apple-mobile-web-capable" content="yes">
        <title>录音并传递给后台</title>
    </head>
    <body>
        <button id="intercomBegin">开始对讲</button>
        <button id="intercomEnd">关闭对讲</button>
    </body>
    <script type="text/javascript">
        var begin = document.getElementById('intercomBegin');
        var end = document.getElementById('intercomEnd');
		
        var ws = null; //实现WebSocket 
        var record = null; //多媒体对象，用来处理音频
 
        function init(rec) {
            record = rec;
        }
		
        //录音对象
        var Recorder = function(stream) {
            var sampleBits = 16; //输出采样数位 8, 16
            var sampleRate = 8000; //输出采样率
            var context = new AudioContext();
            var audioInput = context.createMediaStreamSource(stream);
            var recorder = context.createScriptProcessor(4096, 1, 1);
            var audioData = {
                size: 0, //录音文件长度
                buffer: [], //录音缓存
                inputSampleRate: 48000, //输入采样率
                inputSampleBits: 16, //输入采样数位 8, 16
                outputSampleRate: sampleRate, //输出采样数位
                oututSampleBits: sampleBits, //输出采样率
                clear: function() {
                    this.buffer = [];
                    this.size = 0;
                },
                input: function(data) {
                    this.buffer.push(new Float32Array(data));
                    this.size += data.length;		
                },
                compress: function() { //合并压缩
                    //合并
                    var data = new Float32Array(this.size);
                    var offset = 0;
                    for (var i = 0; i < this.buffer.length; i++) {
                        data.set(this.buffer[i], offset);
                        offset += this.buffer[i].length;
                    }
                    //压缩
                    var compression = parseInt(this.inputSampleRate / this.outputSampleRate);
                    var length = data.length / compression;
                    var result = new Float32Array(length);
                    var index = 0,
                    j = 0;
                    while (index < length) {
                        result[index] = data[j];
                        j += compression;
                        index++;
                    }
                    return result;
                },
                encodePCM: function() { //这里不对采集到的数据进行其他格式处理，如有需要均交给服务器端处理。
                    var sampleRate = Math.min(this.inputSampleRate, this.outputSampleRate);
                    var sampleBits = Math.min(this.inputSampleBits, this.oututSampleBits);
                    var bytes = this.compress();
                    var dataLength = bytes.length * (sampleBits / 8);
                    var buffer = new ArrayBuffer(dataLength);
                    var data = new DataView(buffer);
                    var offset = 0;
                    for (var i = 0; i < bytes.length; i++, offset += 2) {
                    var s = Math.max(-1, Math.min(1, bytes[i]));
                        data.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
                    }
                    return new Blob([data]);
                }
            };
 
            var sendData = function() { //对以获取的数据进行处理(分包)
                var reader = new FileReader();
                reader.onload = e => {
                    var outbuffer = e.target.result;
                    var arr = new Int8Array(outbuffer);
                    if (arr.length > 0) {
                        var tmparr = new Int8Array(1024);
                        var j = 0;
                        for (var i = 0; i < arr.byteLength; i++) {
                            tmparr[j++] = arr[i];
                            if (((i + 1) % 1024) == 0) {
                                ws.send(tmparr);
                                if (arr.byteLength - i - 1 >= 1024) {
                                    tmparr = new Int8Array(1024);
                                } else {
                                    tmparr = new Int8Array(arr.byteLength - i - 1);
                                }
                                j = 0;
                            }
                            if ((i + 1 == arr.byteLength) && ((i + 1) % 1024) != 0) {
                                ws.send(tmparr);
                            }
                        }
                    }
                };
                reader.readAsArrayBuffer(audioData.encodePCM());
                audioData.clear();//每次发送完成则清理掉旧数据
            };
			
            this.start = function() {
                audioInput.connect(recorder);
                recorder.connect(context.destination);
            }
 
            this.stop = function() {
                recorder.disconnect();
            }
 
            this.getBlob = function() {
                return audioData.encodePCM();
            }
 
            this.clear = function() {
                audioData.clear();
            }
			
            recorder.onaudioprocess = function(e) {
                var inputBuffer = e.inputBuffer.getChannelData(0);
                audioData.input(inputBuffer);
                sendData();
            }
        }
        
		
        /*
        * WebSocket
        */
        function useWebSocket() {
            ws = new WebSocket("ws://192.168.2.9:8080/websocket");
            ws.binaryType = 'arraybuffer'; //传输的是 ArrayBuffer 类型的数据
            ws.onopen = function() {
                console.log('握手成功');
                if (ws.readyState == 1) { //ws进入连接状态，则每隔500毫秒发送一包数据
                    record.start();
                }
            };
			
            ws.onmessage = function(msg) {
                console.info(msg)
            }
			
            ws.onerror = function(err) {
                console.info(err)
            }
        }
		
        /*
        * 开始对讲
        */
        begin.onclick = function() {
            navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia;
            if (!navigator.getUserMedia) {
                alert('浏览器不支持音频输入');
            } else {
                navigator.getUserMedia({
                audio: true
            },
            function(mediaStream) {
                init(new Recorder(mediaStream));
                console.log('开始对讲');
                useWebSocket();
            },
            function(error) {
                console.log(error);
                switch (error.message || error.name) {
                    case 'PERMISSION_DENIED':  
                    case 'PermissionDeniedError':  
                        console.info('用户拒绝提供信息。');  
                        break;  
                    case 'NOT_SUPPORTED_ERROR':  
                    case 'NotSupportedError':  
                        console.info('浏览器不支持硬件设备。');  
                        break;  
                    case 'MANDATORY_UNSATISFIED_ERROR':  
                    case 'MandatoryUnsatisfiedError':  
                        console.info('无法发现指定的硬件设备。');  
                        break;  
                        default:  
                        console.info('无法打开麦克风。异常信息:' + (error.code || error.name));  
                        break;  
                        }  
                    }
                )
            }
        }
 
        /*
        * 关闭对讲
        */
        end.onclick = function() {
            if (ws) {
                ws.close();
                record.stop();
                console.log('关闭对讲以及WebSocket');
            }
        }
    </script>
</html>