add displaying video stream using sse

api/index.js

@@ -18,12 +18,29 @@ var ws;
 var api_connected = false;
 
 function send_events_to_clients(data) {
-//   console.log("sending events to clients");
+  //   console.log("sending events to clients");
-    sse_clients.forEach((client) => {
+  sse_clients.forEach((client) => {
-        client.response.write("event: message\n");
+    client.response.write("event: message\n");
-        client.response.write("data:" + JSON.stringify(data) + "\n\n")
+    client.response.write("data:" + JSON.stringify(data) + "\n\n");
-    }
+  });
-  );
+}
+
+function send_image_data_to_clients(frameData) {
+  sse_clients.forEach((client) => {
+    // Create a Blob from the frame data
+    const blob = new Blob([frameData], { type: "image/jpeg" });
+
+    // Set the SSE event name as 'message'
+    client.response.write("event: message\n");
+
+    // Create an object URL from the Blob
+    const objectURL = URL.createObjectURL(blob);
+
+    // Set the SSE event data as the object URL
+    client.response.write(
+      "data: " + JSON.stringify({ image: objectURL }) + "\n\n"
+    );
+  });
 }
 
 function handle_sse_client(request, response, next) {
@@ -50,25 +67,6 @@ function handle_sse_client(request, response, next) {
   });
 }
 
-const getSizeInBytes = obj => {
-    let str = null;
-    if (typeof obj === 'string') {
-      // If obj is a string, then use it
-      str = obj;
-    } else {
-      // Else, make obj into a string
-      str = JSON.stringify(obj);
-    }
-    // Get the length of the Uint8Array
-    const bytes = new TextEncoder().encode(str).length;
-    return bytes;
-};
-  
-const logSizeInBytes = (description, obj) => {
-    const bytes = getSizeInBytes(obj);
-    console.log(`${description} is approximately ${bytes} B`);
-};
-
 var connect_to_api = function () {
   console.log("Connecting to API");
   ws = new WebSocket("ws://10.100.0.40:9001/");
@@ -80,15 +78,15 @@ var connect_to_api = function () {
 
   ws.on("message", function message(message) {
     try {
-        var msg = JSON.parse(message);
+      var msg = JSON.parse(message);
-        if (msg.type != "IMAGE") {
+      if (msg.type != "IMAGE") {
-            send_events_to_clients(msg);
+        send_events_to_clients(msg);
-        } else {
+      } else {
-            console.log("got image");
+        console.log("got image");
-        }
+      }
     } catch (error) {
-        console.log("could not parse as json, must be bytes");
+      console.log("could not parse as json, must be bytes");
-        
+      send_image_data_to_clients(message);
     }
   });
 

api/views/index.ejs

@@ -14,7 +14,7 @@
     <div class="video">
         <div class="mainvideo">
             <p>Camera view:</p>
-            <canvas id="msg" width="960" height="720" style="display: inline-block; border: 1px solid blue;"></canvas>
+            <img id="msg" style="border: 1px solid blue; width: 960px;"></img>
             <div id="connectedbuttons">
                 <div id="connectedstatus">
                     <p id="connectedlabel">Connected: <%- api_connected %></p>
@@ -59,7 +59,7 @@
 </body>
 
 <script>
-    // var update_status = setInterval(update_status, 1000);
+    var update_status = setInterval(update_status, 1000);
     assign_button_callbacks();
     window.onload = function () {
         const events = new EventSource("/events");
@@ -69,10 +69,19 @@
         }
 
         events.onmessage = (event) => {
-            //TODO handling status, failsafe etc
+            console.log("MESSAGE RECEIVED");
-            const parsedData = JSON.parse(event.data);
+            const eventData = JSON.parse(event.data);
-            console.log("RECEIVED EVENT");
+
-            console.log(parsedData);
+            // Check if the event contains image data
+            if (eventData.image) {
+                const imageURL = eventData.image;
+
+                // Process the received image URL
+                // For example, you can set it as the source of an <img> element
+                const imgElement = document.getElementById('image-element');
+                imgElement.src = imageURL;
+            }
+            // console.log(parsedData);
         }
     };
 
@@ -183,51 +192,29 @@
     }
 
     function connect() {
-        console.log("connect");
+        var received = false;
-        socket = new WebSocket("ws://10.0.100.40:9001/");
+        var xhr = new XMLHttpRequest();
-        let msg = document.getElementById("msg");
+        xhr.open("GET", "/connect", true);
-        socket.addEventListener('open', (e) => {
+        xhr.onreadystatechange = function () {
-            console.log("open");
+            if (this.status == 200) {
-        });
+                console.log(this.responseText);
-        socket.addEventListener('message', (e) => {
+                if (this.responseText.length > 0) {
-            let ctx = msg.getContext("2d");
+                    var status = JSON.parse(this.responseText);
-            let image = new Image();
+                    // console.log(status)
-            image.src = URL.createObjectURL(e.data);
+                    document.getElementById("connectedlabel").innerHTML = "Connected: true";
-            image.addEventListener("load", (e) => {
+                    document.getElementById("connectbutton").disabled = true;
-                ctx.drawImage(image, 0, 0, msg.width, msg.height);
+                }
-            });
+            } else {
-        });
+                console.log("error");
+                document.getElementById("connectedlabel").innerHTML = "Connected: false";
+                if (!received) {
+                    alert("Could not connect to API!");
+                    received = true;
 
-        // ws.on("error", function error(err) {
+                }
-        //     console.log("there was an error");
+            }
-        //     console.error("error: " + err);
+        };
-        //     received_error = true;
+        xhr.send();
-        // });
-
-        // var received = false;
-        // var xhr = new XMLHttpRequest();
-        // xhr.open("GET", "/connect", true);
-        // xhr.onreadystatechange = function () {
-        //     if (this.status == 200) {
-        //         console.log(this.responseText);
-        //         if (this.responseText.length > 0) {
-        //             var status = JSON.parse(this.responseText);
-        //             // console.log(status)
-        //             document.getElementById("connectedlabel").innerHTML = "Connected: true";
-        //             document.getElementById("connectbutton").disabled = true;
-        //         }
-        //     } else {
-        //         console.log("error");
-        //         document.getElementById("connectedlabel").innerHTML = "Connected: false";
-        //         if (!received)
-        //         {
-        //             alert("Could not connect to API!");
-        //             received = true;
-
-        //         }
-        //     }
-        // };
-        // xhr.send();
     }
     // window onload function die elke seconde een request doet om te kijken of er al nieuwe foto is
     // function die elke 100 ms een request doet om de status te updaten

src/api_communication/api_communication/api_listener.py

@@ -103,6 +103,7 @@ class ApiListener(Node):
                     frame = cv2.resize(frame,(640,480))
                     encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 65]
                     man = cv2.imencode('.jpg', frame, encode_param)[1]
+                    self.get_logger().info('Sending video')
                     self.message_queue.append(man.tobytes())
 
             except Exception as e: