Panaudia delivers spatial audio over a standard WebRTC connection. The client provides a mono audio stream and gets back a stereo binaural stream.
WebRTC is a well established, widely adopted, and robust protocol for sending and receiving low latency audio over IP, however, not always simple to work with. If you want to add Panaudia to a web project we recommend using the JavaScript SDK which wraps this API and provides a simpler interface.
Our roadmap includes similar SDKs for Unity/C#, iOS/Swift, and Android/Kotlin, but they are not ready yet. If you can't wait for us to write the SDK for your language of choice you can use this WebRTC API directly anywhere that supports WebRTC.
A full explanation of how to use WebRTC is beyond the scope of this guide, but if you are already comfortable using WebRTC this guide should help you integrate Panaudia into your code. Below is a walk through of how to connect including snippets taken from the JavaScript SDK to help illustrate how our signalling and data channels work.
Access to Spaces is controlled by Tickets, these are JWT access tokens. See the Tickets page for how to create them.
Every user will need their own individual Ticket to connect to a Space
The first thing a client needs to do is present their Ticket to the gateway to find the server for the Space:
GET https://panaudia.com/gateway?ticket={ ticket }If the Ticket is valid they will get back the connection url for joining the space, something like this:
{
"status": "ok",
"url": "wss://eem.panaudia.com/join"
}Create an RTC peer connection
pc = new RTCPeerConnection({
iceServers: [
{urls: 'stun:stun.l.google.com:19302'},
{urls: 'stun:stun.l.google.com:5349'},
{urls: 'stun:stun1.l.google.com:3478'},
],
});Configure a callback for onicecandidate
pc.onicecandidate = (e) => {
if (e.candidate && e.candidate.candidate !== '') {
let data = JSON.stringify(e.candidate);
ws.send(JSON.stringify({event: 'candidate', data: data}));
}
};And one for ontrack that will add incoming audio tracks to a player:
pc.ontrack = function (event) {
// console.log(event)
connectionStatusCallback('connected', 'Connected');
let el = document.createElement(event.track.kind);
el.srcObject = event.streams[0];
el.autoplay = true;
el.controls = true;
el.id = 'panaudia-player';
document.getElementById(domPlayerParentId).prepend(el);
event.track.onmute = function (event) {
el.play();
};
event.streams[0].onremovetrack = ({track}) => {
if (el.parentNode) {
el.parentNode.removeChild(el);
}
};
};Our WebRTC signalling uses WebSockets.
Connect using the connection url you discovered above and your Ticket:
ws = new WebSocket(wss://eem.panaudia.com/join?ticket={ ticket });Panaudia will send you an offer SDP over websockets that will look something like this:
v=0
o=- 1913323179848037314 1734611914 IN IP4 0.0.0.0
s=-
t=0 0
a=msid-semantic:WMS*
a=fingerprint:sha-256 FA:A3:4C:CE:6C:CA:0F:48:59:B2:BC:C0:A8:4D:D3:71:0F:84:61:86:54:6D:B7:12:BD:73:9A:6F:2F:09:5B:32
a=extmap-allow-mixed
a=group:BUNDLE 0 1
m=audio 9 UDP/TLS/RTP/SAVPF 111
c=IN IP4 0.0.0.0
a=setup:actpass
a=mid:0
a=ice-ufrag:UOsMromXbcGNBWbn
a=ice-pwd:nEcAbEhvXKrQcpUDarKSkibyASIwdtIt
a=rtcp-mux
a=rtcp-rsize
a=rtpmap:111 opus/48000/2
a=fmtp:111 stereo=1; sprop-stereo=1; minptime=10; maxaveragebitrate=48000; useinbandfec=1
a=ssrc:3262735853 cname:panaudia
a=ssrc:3262735853 msid:panaudia audio
a=ssrc:3262735853 mslabel:panaudia
a=ssrc:3262735853 label:audio
a=msid:panaudia audio
a=sendrecv
m=application 9 UDP/DTLS/SCTP webrtc-datachannel
c=IN IP4 0.0.0.0
a=setup:actpass
a=mid:1
a=sendrecv
a=sctp-port:5000
a=ice-ufrag:UOsMromXbcGNBWbn
a=ice-pwd:nEcAbEhvXKrQcpUDarKSkibyASIwdtItSet this offer as your remote description, and then create an answer, set it as your local description, and then send it to send back to Panaudia.
We munge the answer to force add stereo=1; sprop-stereo=1; to the opus fmtp line. This expresses a preference to both send and receive stereo.
await pc.setRemoteDescription(offer);
let answer = await pc.createAnswer();
answer.sdp = answer.sdp.replace('a=fmtp:111 ', 'a=fmtp:111 stereo=1; sprop-stereo=1; ');
await pc.setLocalDescription(answer);
let data = JSON.stringify(answer);
ws.send(JSON.stringify({event: 'answer', data: data}));If everything worked correctly audio will start flowing in both directions.
Every client must also use a data channel called state to send, and optionally receive, changes to client state:
pc.ondatachannel = (ev) => {
let receiveChannel = ev.channel;
...
if (receiveChannel.label === "state") {
stateDataChannel = receiveChannel;
receiveChannel.onopen = () => {
connectionStatusCallback('data_connected', 'Data channel connected');
};
receiveChannel.onmessage = (msg) => {
if (msg.data instanceof ArrayBuffer) {
let state = PanaudiaNodeState.fromDataBuffer(msg.data);
if (stateCallback !== undefined) {
stateCallback(state.asWebGLCoordinates());
}
if (ambisonicStateCallback !== undefined) {
ambisonicStateCallback(state);
}
} else {
PanaudiaNodeState.fromBlobAsWeb(msg.data, stateCallback);
}
};
}
};Messages in this channel are packed binary buffers 48 bytes long in this format:
When the client moves within the virtual space you must update the server by sending a message giving values for x, y, z, yaw, pitch & roll. For messages you send in this channel you can ignore uuid, volume and gone.
toDataBuffer(x, y, z, yaw, pitch, roll) {
const buffer = new ArrayBuffer(48);
const view = new DataView(buffer);
view.setFloat32(16, x, true);
view.setFloat32(20, y, true);
view.setFloat32(24, z, true);
view.setFloat32(28, yaw, true);
view.setFloat32(32, pitch, true);
view.setFloat32(36, roll, true);
return buffer;
}
stateDataChannel.send(toDataBuffer(x, y, z, yaw, pitch, roll));If you have set data=true in the connection request the Space server will send you messages in this channel giving the locations of all the other clients connected to the server. These messages will also include the uuid of the other client, their current volume and gone will indicate if they have left the server.
const view = new DataView(buffer);
const a = ('0000000000000000' + view.getBigUint64(0, false).toString(16)).slice(-16);
const b = ( '0000000000000000' + view.getBigUint64(8, false).toString(16)).slice(-16);
const guid = `${a.slice(0, 8)}-${a.slice(8, 12)}-${a.slice(12, 16)}-${b.slice(0, 4)}-${b.slice(4, 16)}`;
const x = view.getFloat32(16, true);
const y = view.getFloat32(20, true);
const z = view.getFloat32(24, true);
const yaw = view.getFloat32(28, true);
const pitch = view.getFloat32(32, true);
const roll = view.getFloat32(36, true);
const volume = view.getFloat32(40, true);
const gone = view.getInt32(44, true);If you have set data=true you can also listen to a second data channel called attributes:
pc.ondatachannel = (ev) => {
let receiveChannel = ev.channel;
if (attributesCallback !== undefined && receiveChannel.label === "attributes") {
receiveChannel.onmessage = (msg) => {
let attributes = PanaudiaNodeAttributes.fromJson(msg.data);
if (!attributes) {
return log('failed to parse attributes');
}
attributesCallback(attributes);
};
}
...
};You don't send any messages in this channel, but you will be sent json messages giving custom attributes describing the other clients in the Space:
{
"uuid": "957b3369-d724-42d8-8844-a6ace36b5a1a",
"name": "Paul",
"ticket": {},
"connection": {}
}Ticket will hold any custom attributes that were written into their Ticket and connection will hold any given in their connection request.
You can set the custom attributes for this client by adding attributes to the connection url query string, any values other than ticket, data, x, y, z, yaw, pitch & roll will be gathered and distributed to other clients.
You can optionally set the initial location of the client in virtual space as part of the query when connecting. These are the attrs used:
These coordinates use the native ambisonic coordinate system used by Panaudia.
The connection url might then look something like this:
wss://eu1.panaudia.com/join?x=0.5&y=0.5&z=0.5&yaw=45&pitch=30&roll=0%ticket=...