upgrade packages, improvement on ICE flow

adalkiran · Sep 19, 2024 · c2c076b · c2c076b
1 parent d8d5434
commit c2c076b
Show file tree

Hide file tree

Showing 10 changed files with 121 additions and 59 deletions.
diff --git a/backend/Dockerfile b/backend/Dockerfile
@@ -1,6 +1,6 @@
 # See: https://levelup.gitconnected.com/debugging-go-inside-docker-using-visual-studio-code-and-remote-containers-5c3724fe87b9
 # See for available variants: https://hub.docker.com/_/golang?tab=tags
-ARG VARIANT=1.20.2-bullseye
+ARG VARIANT=1.23.0-bookworm
 FROM golang:${VARIANT}
 
 COPY entrypoint.sh entrypoint-dev.sh /

diff --git a/backend/src/sdp/sdp.go b/backend/src/sdp/sdp.go
@@ -72,11 +72,18 @@ func ParseSdpOfferAnswer(offer map[string]interface{}) *SdpMessage {
 		mediaItem := mediaItemObj.(map[string]interface{})
 		//mediaId := mediaItem["mid"].(float64)
 		sdpMedia.Type = MediaType(mediaItem["type"].(string))
-		candidates := mediaItem["candidates"].([]interface{})
+		candidates := make([]interface{}, 0)
+		if mediaItem["candidates"] != nil {
+			candidates = mediaItem["candidates"].([]interface{})
+		}
 		sdpMedia.Ufrag = mediaItem["iceUfrag"].(string)
 		sdpMedia.Pwd = mediaItem["icePwd"].(string)
 		//iceOptions := mediaItem["iceOptions"].(string)
-		fingerprint := mediaItem["fingerprint"].(map[string]interface{})
+		fingerprintRaw, ok := mediaItem["fingerprint"]
+		if !ok {
+			fingerprintRaw = offer["fingerprint"]
+		}
+		fingerprint := fingerprintRaw.(map[string]interface{})
 		sdpMedia.FingerprintType = FingerprintType(fingerprint["type"].(string))
 		sdpMedia.FingerprintHash = fingerprint["hash"].(string)
 		//direction := mediaItem["direction"].(string)
@@ -109,7 +116,7 @@ func GenerateSdpOffer(iceAgent *agent.ServerAgent) *SdpMessage {
 		})
 	}
 	offer := &SdpMessage{
-		SessionID: "a_sessId",
+		SessionID: "1234",
 		MediaItems: []SdpMedia{
 			{
 				MediaId:  0,

diff --git a/backend/src/transcoding/vp8.go b/backend/src/transcoding/vp8.go
@@ -105,10 +105,19 @@ func (d *VP8Decoder) Run() {
 	}
 }
 
+func (d *VP8Decoder) safeEncodeJpeg(buffer *bytes.Buffer, img *vpx.Image) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("error: %v, image width: %d, height: %d", r, img.W, img.H)
+		}
+	}()
+	return jpeg.Encode(buffer, img.ImageYCbCr(), nil)
+}
+
 func (d *VP8Decoder) saveImageFile(img *vpx.Image) (string, error) {
 	fileCount++
 	buffer := new(bytes.Buffer)
-	if err := jpeg.Encode(buffer, img.ImageYCbCr(), nil); err != nil {
+	if err := d.safeEncodeJpeg(buffer, img); err != nil {
 		return "", fmt.Errorf("jpeg Encode Error: %s", err)
 	}
 

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -1,4 +1,3 @@
-version: '3'
 services:
   backend:
     entrypoint: /entrypoint-dev.sh
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,13 +1,12 @@
-version: '3'
 services:
   ui:
     image: webrtc-nuts-and-bolts/ui
     container_name: webrtcnb-ui
     build:
       context: ui # Dockerfile location
       args:
-        # See for available variants: https://github.com/microsoft/vscode-dev-containers/tree/main/containers/typescript-node
-        - VARIANT:18-bullseye
+        # See for available variants: https://github.com/devcontainers/images/tree/main/src/typescript-node
+        - VARIANT:22-bookworm
     volumes:
       # Mount the root folder that contains .git
       - "./ui:/workspace:cached"
@@ -21,7 +20,7 @@ services:
       context: backend # Dockerfile location
       args:
         # See for available variants: https://hub.docker.com/_/golang?tab=tags
-        - VARIANT:1.20.2-bullseye
+        - VARIANT:1.23.0-bookworm
     # See: https://code.visualstudio.com/docs/remote/create-dev-container#_set-up-a-folder-to-run-in-a-container
     # [Optional] Required for ptrace-based debuggers like C++, Go, and Rust
     cap_add:

diff --git a/docs/00-INFRASTRUCTURE.md b/docs/00-INFRASTRUCTURE.md
@@ -14,7 +14,7 @@ When you run the docker-compose.yml file individually (for production mode) or d
     build:
       context: ui # Dockerfile location
       args:
-        - VARIANT:18-bullseye
+        - VARIANT:22-bookworm
     volumes:
       # Mount the root folder that contains .git
       - "./ui:/workspace:cached"
@@ -26,8 +26,8 @@ When you run the docker-compose.yml file individually (for production mode) or d
 <sup>Related part of [ui/Dockerfile](../ui/Dockerfile):</sup>
 
 ```dockerfile
-ARG VARIANT=18-bullseye
-FROM mcr.microsoft.com/vscode/devcontainers/typescript-node:0-${VARIANT}
+ARG VARIANT=22-bookworm
+FROM mcr.microsoft.com/vscode/devcontainers/typescript-node:${VARIANT}
 
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive
 
@@ -36,7 +36,7 @@ WORKDIR /workspace
 ENTRYPOINT yarn install && npm run start
 ```
 
-This file inherits from *mcr.microsoft.com/vscode/devcontainers/typescript-node:18-bullseye* image which come up with an environment includes NodeJS, Webpack, Webpack Dev Server, TypeScript, over Debian "Bullseye" Linux Distribution. We don't need to install related things manually.
+This file inherits from *mcr.microsoft.com/vscode/devcontainers/typescript-node:22-bookworm* image which come up with an environment includes NodeJS, Webpack, Webpack Dev Server, TypeScript, over Debian "bookworm" Linux Distribution. We don't need to install related things manually.
 
 While building the custom image (once):
 
@@ -63,7 +63,7 @@ While every time of the container starting up:
     build:
       context: backend # Dockerfile location
       args:
-        - VARIANT:1.20.2-bullseye
+        - VARIANT:1.23.0-bookworm
     # [Optional] Required for ptrace-based debuggers like C++, Go, and Rust
     cap_add:
       - SYS_PTRACE
@@ -81,7 +81,7 @@ While every time of the container starting up:
 <sup>Related part of [backend/Dockerfile](../backend/Dockerfile):</sup>
 
 ```dockerfile
-ARG VARIANT=1.20.2-bullseye
+ARG VARIANT=1.23.0-bookworm
 FROM golang:${VARIANT}
 
 COPY entrypoint.sh entrypoint-dev.sh /
@@ -95,7 +95,7 @@ WORKDIR /workspace
 ENTRYPOINT "/entrypoint.sh"
 ```
 
-This file inherits from *golang:1.20.2-bullseye* image which come up with an environment includes Go language support, libraries for processing VP8 (video) and OPUS (audio) encoding, on Debian "Bullseye" Linux Distribution. We don't need to install related things manually.
+This file inherits from *golang:1.23.0-bookworm* image which come up with an environment includes Go language support, libraries for processing VP8 (video) and OPUS (audio) encoding, on Debian "bookworm" Linux Distribution. We don't need to install related things manually.
 
 While building the custom image (once):
 

diff --git a/docs/03-FIRST-CLIENT-COMES-IN.md b/docs/03-FIRST-CLIENT-COMES-IN.md
@@ -49,26 +49,39 @@ We assign some event handlers for some events, which we will discuss further. We
             iceServers: [{ urls: 'stun:stun.l.google.com:19302' }]
         });
         result.onicecandidate = (e: RTCPeerConnectionIceEvent) => {
-            if ((<any>e.target).iceGatheringState != 'complete') {
-                return;
+            if (e.candidate === null) {
+                console.log('onicecandidate: localSessionDescription:\n', (<any>e.target).localDescription);
+            } else {            
+                console.log('New ICE candidate:', e.candidate);
             }
-            console.log('onicecandidate', e.candidate, '\n', e);
-            const parsedSdp: sdpTransform.SessionDescription = sdpTransform.parse(this.localConnection.localDescription.sdp);
-            this.localDescriptionChanged(parsedSdp);
         };
         result.addEventListener('track', e => {
             console.log('onTrack', e);
         });
         result.onicecandidateerror = (e: Event) => {
-            console.log('onicecandidateerror', e);
+            console.log("onicecandidateerror", "candidate address:", (<any>e).hostCandidate ?? '', "error text:", (<any>e).errorText ?? '', e);
         };
         result.onconnectionstatechange = (e: Event) => {
             console.log('onconnectionstatechange', (<any>e.target).connectionState, '\n', e);
         };
         result.oniceconnectionstatechange = (e: Event) => {
-            console.log('oniceconnectionstatechange', (<any>e.target).iceConnectionState, '\n', e);
-            if ((<any>e.target).iceConnectionState == 'disconnected') {
+            const peerConnection = <any>e.target;
+            this.updateStatus(peerConnection.iceConnectionState);
+            console.log('oniceconnectionstatechange', peerConnection.iceConnectionState, '\n', e);
+            if (peerConnection.iceConnectionState == 'disconnected') {
                 this.stop(true);
+            } else if (peerConnection.iceConnectionState === 'connected' || peerConnection.iceConnectionState === 'completed') {
+                // Get the stats for the peer connection
+                peerConnection.getStats().then((stats: any) => {                
+                    stats.forEach((report: any) => {
+                        if (report.type === 'candidate-pair' && report.state === 'succeeded') {
+                            const localCandidate = stats.get(report.localCandidateId);
+                            const remoteCandidate = stats.get(report.remoteCandidateId);
+                            console.log('Succeded Local Candidate:', report.localCandidateId, 'address:', localCandidate?.address, 'object:', localCandidate);
+                            console.log('Succeded Remote Candidate:', report.remoteCandidateId, 'address:', remoteCandidate?.address, 'object:', remoteCandidate);
+                        }
+                    });
+                });
             }
         };
         result.onicegatheringstatechange = (e: Event) => {
@@ -114,6 +127,8 @@ The button click will call "rtc.start()" function which:
 
 ```ts
     start() {
+        this.updateStatus("starting...");
+        this.localConnection = this.createLocalPeerConnection();
         return this.createLocalTracks()
             .then(stream => {
                 stream.getTracks().forEach(track => {
@@ -235,7 +250,12 @@ The "Welcome message" coming from the server is processed by switch case for "We
 
 ```ts
     this.ws.onmessage = (message) => {
-        const data = message.data ? JSON.parse(message.data) : null;
+        let data = null;
+        try {
+            data = message.data ? JSON.parse(message.data) : null;
+        } catch {
+            // Do nothing
+        }            
         console.log('Received from WS:', message.data);
         if (!data) {
             return;
@@ -335,7 +355,7 @@ The "SDP Offer message" coming from the server is processed by switch case for "
                     username: 'a_user',
                     sessionId: sdpOffer.sessionId,
                     sessionVersion: 2,
-                    netType: 'udp',
+                    netType: 'IN',
                     ipVer: 4,
                     address: '127.0.0.1'
                 },
@@ -408,6 +428,10 @@ The "SDP Offer message" coming from the server is processed by switch case for "
             return this.localConnection.createAnswer()
                 .then((answer: RTCSessionDescriptionInit) => {
                     console.log('answer', answer.type, answer.sdp);
+                    const parsedSdp: sdpTransform.SessionDescription = sdpTransform.parse(
+                        answer.sdp
+                    );
+                    this.sendSdpToSignaling(parsedSdp);
                     this.localConnection.setLocalDescription(answer);
                 });
         });

diff --git a/ui/Dockerfile b/ui/Dockerfile
@@ -1,8 +1,7 @@
-# See here for image contents: https://github.com/microsoft/vscode-dev-containers/blob/main/containers/typescript-node/.devcontainer/base.Dockerfile
-# See for available variants: https://github.com/microsoft/vscode-dev-containers/tree/main/containers/typescript-node
-# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT=18-bullseye
-FROM mcr.microsoft.com/vscode/devcontainers/typescript-node:0-${VARIANT}
+# See here for image contents: https://github.com/devcontainers/images/blob/main/src/typescript-node/.devcontainer/Dockerfile
+# See for available variants: https://github.com/devcontainers/images/tree/main/src/typescript-node
+ARG VARIANT=22-bookworm
+FROM mcr.microsoft.com/vscode/devcontainers/typescript-node:${VARIANT}
 
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive
 #     && apt-get -y install --no-install-recommends <your-package-list-here>

diff --git a/ui/src/app.ts b/ui/src/app.ts
@@ -8,33 +8,48 @@ class RTC {
 
     constructor() {
         this.localConnection = this.createLocalPeerConnection();
+        this.localConnection = null;
     }
 
     createLocalPeerConnection() {
         const result = new RTCPeerConnection({
             iceServers: [{ urls: 'stun:stun.l.google.com:19302' }]
         });
         result.onicecandidate = (e: RTCPeerConnectionIceEvent) => {
-            if ((<any>e.target).iceGatheringState != 'complete') {
-                return;
+            if (e.candidate === null) {
+                console.log('onicecandidate: localSessionDescription:\n', (<any>e.target).localDescription);
+            } else {            
+                console.log('New ICE candidate:', e.candidate);
             }
-            console.log('onicecandidate', e.candidate, '\n', e);
-            const parsedSdp: sdpTransform.SessionDescription = sdpTransform.parse(this.localConnection.localDescription.sdp);
-            this.localDescriptionChanged(parsedSdp);
         };
         result.addEventListener('track', e => {
             console.log('onTrack', e);
         });
         result.onicecandidateerror = (e: Event) => {
-            console.log('onicecandidateerror', e);
+            console.log("onicecandidateerror", "candidate address:", (<any>e).hostCandidate ?? '', "error text:", (<any>e).errorText ?? '', e);
         };
         result.onconnectionstatechange = (e: Event) => {
+            this.updateStatus((<any>e.target).connectionState);
             console.log('onconnectionstatechange', (<any>e.target).connectionState, '\n', e);
         };
         result.oniceconnectionstatechange = (e: Event) => {
-            console.log('oniceconnectionstatechange', (<any>e.target).iceConnectionState, '\n', e);
-            if ((<any>e.target).iceConnectionState == 'disconnected') {
+            const peerConnection = <any>e.target;
+            this.updateStatus(peerConnection.iceConnectionState);
+            console.log('oniceconnectionstatechange', peerConnection.iceConnectionState, '\n', e);
+            if (peerConnection.iceConnectionState == 'disconnected') {
                 this.stop(true);
+            } else if (peerConnection.iceConnectionState === 'connected' || peerConnection.iceConnectionState === 'completed') {
+                // Get the stats for the peer connection
+                peerConnection.getStats().then((stats: any) => {                
+                    stats.forEach((report: any) => {
+                        if (report.type === 'candidate-pair' && report.state === 'succeeded') {
+                            const localCandidate = stats.get(report.localCandidateId);
+                            const remoteCandidate = stats.get(report.remoteCandidateId);
+                            console.log('Succeded Local Candidate:', report.localCandidateId, 'address:', localCandidate?.address, 'object:', localCandidate);
+                            console.log('Succeded Remote Candidate:', report.remoteCandidateId, 'address:', remoteCandidate?.address, 'object:', remoteCandidate);
+                        }
+                    });
+                });
             }
         };
         result.onicegatheringstatechange = (e: Event) => {
@@ -49,16 +64,6 @@ class RTC {
         return result;
     }
 
-    createOffer(): Promise<sdpTransform.SessionDescription> {
-        return this.localConnection.createOffer()
-            .then(sdp => {
-                this.localConnection.setLocalDescription(sdp);
-                const parsedSdp: sdpTransform.SessionDescription = sdpTransform.parse(sdp.sdp);
-                console.log('setLocalDescription', 'type:', sdp.type, 'sdp:\n', parsedSdp);
-                return parsedSdp;
-            })
-    }
-
     createLocalTracks(): Promise<MediaStream> {
         return navigator.mediaDevices.getUserMedia({
             video: {
@@ -69,6 +74,8 @@ class RTC {
     }
 
     start() {
+        this.updateStatus("starting...");
+        this.localConnection = this.createLocalPeerConnection();
         return this.createLocalTracks()
             .then(stream => {
                 stream.getTracks().forEach(track => {
@@ -89,19 +96,18 @@ class RTC {
             localTrack.enabled = false;
             localTrack.stop();
         });
-        if (closeConnection) {
+        if (closeConnection && this.localConnection) {
+            const updateStatusRequired = !(this.localConnection.iceConnectionState == "disconnected");
             this.localConnection.close();
-            //Recreate a new RTCPeerConnection which is in "stable" signaling state.
-            this.localConnection = this.createLocalPeerConnection();
+            this.localConnection = null;
+            if (updateStatusRequired) {
+                this.updateStatus("closed");
+            }
         }
         this.localTracks = [];
         console.log('Stopping tracks. closeConnection: ', closeConnection);
     }
 
-    localDescriptionChanged(parsedSdp: sdpTransform.SessionDescription) {
-        this.sendSdpToSignaling(parsedSdp);
-    }
-
     sendSdpToSignaling(parsedSdp: sdpTransform.SessionDescription) {
         console.log('sendSdpToSignaling', parsedSdp);
         signaling.ws.send(JSON.stringify({type: "SdpOfferAnswer", data: parsedSdp}));
@@ -116,10 +122,18 @@ class RTC {
             return this.localConnection.createAnswer()
                 .then((answer: RTCSessionDescriptionInit) => {
                     console.log('answer', answer.type, answer.sdp);
+                    const parsedSdp: sdpTransform.SessionDescription = sdpTransform.parse(
+                        answer.sdp
+                    );
+                    this.sendSdpToSignaling(parsedSdp);
                     this.localConnection.setLocalDescription(answer);
                 });
         });
     }
+
+    updateStatus(connStatus: string) {
+        $("#LblConnStatus").text("Connection status: " + connStatus);
+    }
 }
 
 class Signaling {
@@ -144,7 +158,12 @@ class Signaling {
         };
 
         this.ws.onmessage = (message) => {
-            const data = message.data ? JSON.parse(message.data) : null;
+            let data = null;
+            try {
+                data = message.data ? JSON.parse(message.data) : null;
+            } catch {
+                // Do nothing
+            }            
             console.log('Received from WS:', message.data);
             if (!data) {
                 return;
@@ -165,7 +184,7 @@ class Signaling {
                         username: 'a_user',
                         sessionId: sdpOffer.sessionId,
                         sessionVersion: 2,
-                        netType: 'udp',
+                        netType: "IN",
                         ipVer: 4,
                         address: '127.0.0.1'
                     },