Proxy stream

Overview

Applies to Text-to-Speech and Speech-to-Text. Examples use Speech-to-Text.

This guide explains how to stream microphone audio from a client to the Soniox WebSocket API through a proxy server.

The client captures audio and sends it over a WebSocket to your proxy server. The proxy opens the Soniox WebSocket with your API key, forwards the audio, and sends the results back to the client.

Use a proxy when you want to inspect, transform, or store audio and transcription data on the server before it reaches the client. The browser never sees a Soniox key, so you need no temporary keys, but every user holds one connection on your server and one from your server to Soniox, and the second counts against your connection limit. If you only want results with the lowest latency, use direct stream instead.

Example

In the following example, we create a proxy HTTP server that:

Listens for incoming WebSocket connections from the client.
Forwards audio data from the client to the WebSocket API.
Relays transcription results back to the client.

The proxy authenticates with your SONIOX_API_KEY, sent in the Authorization header as described in WebSocket authentication. The browser never sees it. The example accepts any client; in production, authenticate your own users before you open a Soniox connection for them.

Python server that will act as a proxy between our client and WebSocket API.

import os
import json
import asyncio
from pathlib import Path

from dotenv import load_dotenv
import websockets
from websockets.datastructures import Headers
from websockets.http11 import Response

load_dotenv()

INDEX_HTML = Path(__file__).parent / "templates" / "index.html"


async def handle_client(websocket):
    print("Browser client connected")

    # create a message queue to store client messages received before
    # Soniox WebSocket API connection is ready, so we don't loose any
    message_queue = []
    soniox_ws = None
    soniox_ws_ready = False

    async def init_soniox_connection():
        nonlocal soniox_ws, soniox_ws_ready

        try:
            soniox_ws = await websockets.connect(
                "wss://stt-rt.soniox.com/transcribe-websocket",
                additional_headers={
                    "Authorization": f"Bearer {os.getenv('SONIOX_API_KEY')}"
                },
            )
            print("Connected to Soniox STT WebSocket API")

            # Send initial configuration message
            start_message = json.dumps(
                {
                    "audio_format": "auto",
                    "model": "stt-rt-v5",
                    "language_hints": ["en"],
                }
            )
            await soniox_ws.send(start_message)
            print("Sent start message to Soniox")

            # mark connection as ready
            soniox_ws_ready = True

            # process any queued messages
            while len(message_queue) > 0 and soniox_ws_ready:
                data = message_queue.pop(0)
                await forward_data(data)

            # receive messages from Soniox STT WebSocket API
            async for message in soniox_ws:
                try:
                    await websocket.send(message)
                except Exception as e:
                    print(f"Error forwarding Soniox response: {e}")
                    break

        except Exception as e:
            print(f"Soniox WebSocket error: {e}")
            soniox_ws_ready = False
        finally:
            if soniox_ws:
                await soniox_ws.close()
            soniox_ws_ready = False
            print("Soniox WebSocket closed")
            await websocket.close()

    async def forward_data(data):
        try:
            if soniox_ws:
                await soniox_ws.send(data)
        except Exception as e:
            print(f"Error forwarding data to Soniox: {e}")

    # initialize Soniox connection
    soniox_task = asyncio.create_task(init_soniox_connection())

    try:
        # receive messages from browser client
        async for data in websocket:
            if soniox_ws_ready:
                # forward messages instantly
                await forward_data(data)
            else:
                # queue the message to be processed
                # as soon as connection to Soniox STT WebSocket API is ready
                message_queue.append(data)
    except Exception as e:
        print(f"Error with browser client: {e}")
    finally:
        print("Browser client disconnected")
        soniox_task.cancel()
        try:
            await soniox_task
        except asyncio.CancelledError:
            pass


def serve_index(connection, request):
    # WebSocket upgrade: return None so websockets hands it to handle_client
    if request.headers.get("Upgrade", "").lower() == "websocket":
        return None

    # any other request: serve the browser client
    body = INDEX_HTML.read_bytes()
    return Response(200, "OK", Headers({"Content-Type": "text/html"}), body)


async def main():
    port = int(os.getenv("PORT", 8000))
    server = await websockets.serve(
        handle_client, "0.0.0.0", port, process_request=serve_index
    )
    print(f"WebSocket proxy server listening on http://0.0.0.0:{port}")

    await server.wait_closed()


if __name__ == "__main__":
    asyncio.run(main())

Node.js server that will act as a proxy between our client and WebSocket API.

require("dotenv").config();

const WebSocket = require("ws");
const http = require("http");
const fs = require("fs");
const path = require("path");

// serve the browser client; WebSocket upgrades on the same port go to wss below
const server = http.createServer((req, res) => {
  res.writeHead(200, { "Content-Type": "text/html" });
  fs.createReadStream(path.join(__dirname, "templates/index.html")).pipe(res);
});
const wss = new WebSocket.Server({ server });

wss.on("connection", (ws) => {
  console.log("Browser client connected");

  // create a message queue to store client messages received before
  // Soniox WebSocket API connection is ready, so we don't loose any
  const messageQueue = [];

  let sonioxWs = null;
  let sonioxWsReady = false;

  function initSonioxConnection() {
    sonioxWs = new WebSocket("wss://stt-rt.soniox.com/transcribe-websocket", {
      headers: { Authorization: `Bearer ${process.env.SONIOX_API_KEY}` },
    });

    sonioxWs.on("open", () => {
      console.log("Connected to Soniox STT WebSocket API");

      // send initial configuration message
      const startMessage = JSON.stringify({
        audio_format: "auto",
        model: "stt-rt-v5",
        language_hints: ["en"],
      });
      sonioxWs.send(startMessage);
      console.log("Sent start message to Soniox");

      // mark connection as ready
      sonioxWsReady = true;

      // process any queued messages
      while (messageQueue.length > 0 && sonioxWsReady) {
        const { data, isBinary } = messageQueue.shift();
        forwardData(data, isBinary);
      }
    });

    // receive messages from Soniox STT WebSocket API
    sonioxWs.on("message", (data) => {
      // note:
      // at this point we could manipulate and enhance the transcribed data
      try {
        ws.send(data.toString());
      } catch (err) {
        console.error("Error forwarding Soniox response:", err);
      }
    });

    sonioxWs.on("error", (error) => {
      console.log("Soniox WebSocket error:", error);
      sonioxWsReady = false;
    });

    sonioxWs.on("close", (code, reason) => {
      console.log("Soniox WebSocket closed:", code, reason);
      sonioxWsReady = false;
      ws.close();
    });
  }

  // forward message data to Soniox STT WebSocket API
  function forwardData(data, isBinary) {
    try {
      // keep the frame type, so the empty text frame still ends the stream
      sonioxWs.send(data, { binary: isBinary });
    } catch (err) {
      console.error("Error forwarding data to Soniox:", err);
    }
  }

  // initialize Soniox connection
  initSonioxConnection();

  // receive messages from browser client
  ws.on("message", (data, isBinary) => {
    if (sonioxWsReady) {
      // forward messages instantly
      forwardData(data, isBinary);
    } else {
      // queue the message to be processed
      // as soon as connection to Soniox STT WebSocket API is ready
      messageQueue.push({ data, isBinary });
    }
  });

  ws.on("close", () => {
    console.log("Browser client disconnected");
    if (sonioxWs) {
      try {
        sonioxWs.close();
      } catch (err) {
        console.error("Error closing Soniox connection:", err);
      }
    }
  });
});

server.listen(process.env.PORT, () => {
  console.log(
    `WebSocket proxy server listening on http://0.0.0.0:${process.env.PORT}`,
  );
});

Next, we create a basic HTML page as the client (same concept works for any other app framework).

The HTML client:

Connects to the proxy server via WebSocket.
Captures audio stream from the microphone through the MediaRecorder.
Streams audio data to the proxy server.
Receives messages from the proxy server and renders transcribed text into a div.

<!DOCTYPE html>
<html>

<body>
  <h1>Browser proxy stream example</h1>
  <button id="trigger">Start</button>
  <hr />
  <div>
    <span id="final"></span>
    <span id="nonfinal" style="color: gray"></span>
  </div>
  <div id="error"></div>
  <script>
    const finalEl = document.getElementById("final");
    const nonFinalEl = document.getElementById("nonfinal");
    const errorEl = document.getElementById("error");
    const trigger = document.getElementById("trigger");

    let ws;
    let recorder;
    let recorderState = "stopped"; // "stopped" | "starting" | "running" | "stopping"

    trigger.onclick = async () => {
      if (recorderState === "stopped") {
        finalEl.textContent = "";
        nonFinalEl.textContent = "";
        errorEl.textContent = "";
        trigger.textContent = "Starting...";
        recorderState = "starting";

        // get audio stream from user microphone
        const stream = await navigator.mediaDevices.getUserMedia({ audio: true });

        // connect to the proxy server
        ws = new WebSocket(`ws://${location.host}/`);

        ws.onopen = () => {
          recorder = new MediaRecorder(stream);

          recorder.ondataavailable = async (event) => {
            if (event.data.size > 0) {
              // convert the recorded audio chunk (Blob) to raw binary (ArrayBuffer)
              // and send via websocket message
              ws.send(await event.data.arrayBuffer());
            }
          };

          recorder.onstop = () => {
            // send empty string message to tell the Soniox WebSocket API to stop
            ws.send("");
          };

          // start recording, creating data chunks every 120ms
          recorder.start(120);

          recorderState = "running";
          trigger.textContent = "Stop";
        };

        let finalText = "";

        ws.onmessage = (event) => {
          // parse messages received from Node.js server
          const result = JSON.parse(event.data);

          if (result.error_message) {
            errorEl.textContent = `${result.error_message}`;
            return;
          }

          // render the transcript
          let nonFinalText = "";

          for (let token of result.tokens) {
            if (token.is_final) {
              finalText += token.text;
            } else {
              nonFinalText += token.text;
            }
          }

          finalEl.textContent = finalText;
          nonFinalEl.textContent = nonFinalText;
        };

        ws.onerror = (error) => {
          console.error("WebSocket error:", error);
          errorEl.textContent = "WebSocket error, see the console.";
          stopRecording();
        };

        ws.onclose = (event) => {
          console.log("WebSocket connection closed", event.code);
          stopRecording();
        };
      } else if (recorderState === "running") {
        stopRecording();
      }
    };

    function stopRecording() {
      if (recorder) {
        // stop microphone recording properly
        recorder.stop();
        recorder.stream.getTracks().forEach((t) => t.stop());
      }
      trigger.textContent = "Start";
      recorderState = "stopped";
    }
  </script>
</body>

</html>

Run it

Put index.html in a templates/ folder next to the server, and create a .env file:

SONIOX_API_KEY=<your API key>
PORT=8000

pip install websockets python-dotenv
python server.py

npm init -y
npm install ws dotenv
node server.js

Open http://localhost:8000 and click Start.

Proxy stream

Overview

Example

Run it

On this page