add basic audio support with some bugs

index.js

@@ -17,7 +17,8 @@ var bodyParser = require('body-parser');
 var baseRouter = express.Router();
 var fsw = require('fs').promises;
 var fs = require('fs');
-
+var PulseAudio = require('pulseaudio2');
+var pulse = new PulseAudio();
 
 //// Server Paths Main ////
 app.engine('html', require('ejs').renderFile);
@@ -130,6 +131,34 @@ io.on('connection', async function (socket) {
   socket.on('createfolder', createFolder);
 });
 
+//// PCM Audio Wrapper ////
+aio = socketIO(http, {path: SUBFOLDER + 'audio/socket.io'});
+aio.on('connection', async function (socket) {
+  var record;
+  let id = socket.id;
+
+  async function open() {
+    if (record) record.end();
+      record = pulse.createRecordStream({
+                 channels: 2,
+                 rate: 48000,
+                 format: 'S16LE',
+               });
+      record.on('connection', function(){
+        record.on('data', function(chunk) {
+          aio.sockets.to(id).emit('audio', chunk);
+        });
+      });
+  }
+  async function close() {
+    if (record) record.end();
+  }
+
+
+  // Incoming socket requests
+  socket.on('open', open);
+  socket.on('close', close);
+});
 
 // Spin up application on 6900
 app.use(SUBFOLDER, baseRouter);

package.json

@@ -1,11 +1,12 @@
 {
   "name": "kclient",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Kclient is a wrapper for KasmVNC to add functionality to a containerized environment",
   "main": "index.js",
   "dependencies": {
     "ejs": "^3.1.8",
     "express": "^4.18.2",
+    "pulseaudio2": "^0.5.5",
     "socket.io": "^4.6.0"
   },
   "devDependencies": {},

public/css/kclient.css

@@ -69,6 +69,10 @@
   filter: invert(100%) sepia(0%) saturate(0%) hue-rotate(82deg) brightness(105%) contrast(105%);
 }
 
+.icons-selected {
+  background: rgba(0, 0, 0, 0.3);
+}
+
 .icons:hover {
   background: rgba(0, 0, 0, 0.3);
 }

public/css/speaker.svg

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg version="1.0" viewBox="0 0 75 75" xmlns="http://www.w3.org/2000/svg">
+<path d="m39.389 13.769-17.154 14.837h-16.235v19.093h15.989l17.4 15.051v-48.981z" fill="#111" stroke="#111" stroke-linejoin="round" stroke-width="5"/>
+<path d="m48 27.6a19.5 19.5 0 0 1 0 21.4m7.1-28.5a30 30 0 0 1 0 35.6m6.5-42.1a38.8 38.8 0 0 1 0 48.6" fill="none" stroke="#111" stroke-linecap="round" stroke-width="5"/>
+</svg>

public/index.html

@@ -9,13 +9,16 @@
     <link rel="apple-touch-icon" href="icon.png">
     <link rel="manifest" href="manifest.json">
     <link href="public/css/kclient.css" rel="stylesheet">
+    <script src="audio/socket.io/socket.io.js"></script>
+    <script src="public/js/pcm-player.min.js"></script>
   </head>
   <body>
     <!--KasmVNC Iframe-->
     <iframe class="vnc" src="vnc/index.html?resize=remote&clipboard_up=true&clipboard_down=true&clipboard_seamless=true&show_control_bar=true"></iframe>
     <!--LSIO Function Bar-->
     <div id="lsbar">
-      <img class="icons" title="File Manager" src="public/css/files.svg" onclick="toggle('#files')"/>
+      <img class="icons" id="fileButton" title="File Manager" src="public/css/files.svg" onclick="toggle('#files')"/>
+      <img class="icons" id="audioButton" title="Enable Audio" src="public/css/speaker.svg" onclick="audio()"/>
     </div>
     <!--File Browser-->
     <div id="files">

public/js/kclient.js

@@ -58,3 +58,36 @@ function fullscreen() {
     }
   }
 }
+
+// Websocket comms for audio
+var host = window.location.hostname;
+var port = window.location.port;
+var protocol = window.location.protocol;
+var path = window.location.pathname;
+var socket = io(protocol + '//' + host + ':' + port, { path: path + 'audio/socket.io'});
+var player = {};
+
+function audio() {
+  if (('audioCtx' in player) && (player.audioCtx)) {
+    player.destroy();
+    socket.emit('close', '');
+    $('#audioButton').removeClass("icons-selected");
+    return;
+  }
+  socket.emit('open', '');
+  player = new PCMPlayer({
+             encoding: '16bitInt',
+             channels: 2,
+             sampleRate: 48000,
+             flushingTime: 10
+           });
+  $('#audioButton').addClass("icons-selected");
+}
+
+function processAudio(data) {
+  let buf = new Uint16Array(data);
+  player.feed(buf);
+  player.volume(1) ;
+}
+
+socket.on('audio', processAudio);

public/js/pcm-player.min.js

@@ -0,0 +1 @@
+function PCMPlayer(t){this.init(t)}PCMPlayer.prototype.init=function(t){this.option=Object.assign({},{encoding:"16bitInt",channels:1,sampleRate:8e3,flushingTime:1e3},t),this.samples=new Float32Array,this.flush=this.flush.bind(this),this.interval=setInterval(this.flush,this.option.flushingTime),this.maxValue=this.getMaxValue(),this.typedArray=this.getTypedArray(),this.createContext()},PCMPlayer.prototype.getMaxValue=function(){var t={"8bitInt":128,"16bitInt":32768,"32bitInt":2147483648,"32bitFloat":1};return t[this.option.encoding]?t[this.option.encoding]:t["16bitInt"]},PCMPlayer.prototype.getTypedArray=function(){var t={"8bitInt":Int8Array,"16bitInt":Int16Array,"32bitInt":Int32Array,"32bitFloat":Float32Array};return t[this.option.encoding]?t[this.option.encoding]:t["16bitInt"]},PCMPlayer.prototype.createContext=function(){this.audioCtx=new(window.AudioContext||window.webkitAudioContext),this.gainNode=this.audioCtx.createGain(),this.gainNode.gain.value=1,this.gainNode.connect(this.audioCtx.destination),this.startTime=this.audioCtx.currentTime},PCMPlayer.prototype.isTypedArray=function(t){return t.byteLength&&t.buffer&&t.buffer.constructor==ArrayBuffer},PCMPlayer.prototype.feed=function(t){if(this.isTypedArray(t)){t=this.getFormatedValue(t);var e=new Float32Array(this.samples.length+t.length);e.set(this.samples,0),e.set(t,this.samples.length),this.samples=e}},PCMPlayer.prototype.getFormatedValue=function(t){t=new this.typedArray(t.buffer);var e,i=new Float32Array(t.length);for(e=0;e<t.length;e++)i[e]=t[e]/this.maxValue;return i},PCMPlayer.prototype.volume=function(t){this.gainNode.gain.value=t},PCMPlayer.prototype.destroy=function(){this.interval&&clearInterval(this.interval),this.samples=null,this.audioCtx.close(),this.audioCtx=null},PCMPlayer.prototype.flush=function(){if(this.samples.length){var t,e,i,n,a,s=this.audioCtx.createBufferSource(),r=this.samples.length/this.option.channels,o=this.audioCtx.createBuffer(this.option.channels,r,this.option.sampleRate);for(e=0;e<this.option.channels;e++)for(t=o.getChannelData(e),i=e,a=50,n=0;n<r;n++)t[n]=this.samples[i],n<50&&(t[n]=t[n]*n/50),r-51<=n&&(t[n]=t[n]*a--/50),i+=this.option.channels;this.startTime<this.audioCtx.currentTime&&(this.startTime=this.audioCtx.currentTime),console.log("start vs current "+this.startTime+" vs "+this.audioCtx.currentTime+" duration: "+o.duration),s.buffer=o,s.connect(this.gainNode),s.start(this.startTime),this.startTime+=o.duration,this.samples=new Float32Array}};