in the real world of real time:
Data Channels
Hadar Weiss
CTO and Co-Founder, Peer5
Hacker @Sharefest
CTO and Co-Founder, Peer5
Hacker @Sharefest
// twt:whadar // hadar@peer5.com
Again, why is this so
huge
The web meant to be
Decentralized
In reality we are quite centralized due to the fact:
web browsers can only communicate with servers
(HTTP, WebSockets)
WebRTC should make the web fully distributed
cool,
do you haz teh codez?
(actually it's not that simple)
Tasks:
- Security
-
Signalling
- NAT traversal
-
Decide on reliable vs unreliable
-
P2P encapsulation and serialization
-
Browser support and interoperability
-
WebRTC hacks
-
P2P coordination
-
HTTP augmentation
Reliable or Unreliable
(or partially reliable)
SCTP!
(Stream Control Transmission Protocol)
TCP vs UDP vs SCTP
High Performance Browser Networking - by Ilya Grigorik
SCTP Sample
var channel_confs = {
TCPLIKE_CHANNEL:{}, //reliable, ordered
UNORDERED_RELIABLE:{ ordered: false },
ORDERED_PARTIAL_3ATTEMPS: { ordered: true, maxRetransmits: 3},
UNORDERED_PARTIAL_3ATTEMPS: { ordered: false, maxRetransmits: 3},
ORDERED_PARTIAL_1S: { ordered: true, maxRetransmitTime: 1000},
UNORDERED_PARTIAL_1S: { ordered: false, maxRetransmitTime: 1000},
UNORDERED_UNRELIABLE: { ordered: false, maxRetransmits: 0 }
}
var pc = new RTCPeerConnection(servers
) // don't do {RtpDataChannels:true}
var dc = pc.createDataChannel("dc_label", channel_confs.UNORDERED_PARTIAL_3ATTEMPS);
Underlying Protocols
High Performance Browser Networking - by Ilya Grigorik
P2P Multicasting
Why?
Video
Streaming
Streaming
Radio
File Sharing
VOD
live
gaming
Why not just servers
How
Gigantic Mesh!
Matching Algorithm
- Available Chunks
- Location
- ISP
- Local Network
- Available Bandwidth
- Available CPU
- # of connected peers
- Expected peer lifetime
- WebRTC specific
- Browser vendor
- Browser version
- Many other factors you can consider such as network congestion, mobile data plan, share ratio, etc.
More P2P logic
What peers are there in the swarm?
What peers can I connect to?
What parts (chunks) of the resource each peer has?
Which peers are available to send me data?
Do I really get the chunks that I asked for?
Am I getting chunks in the best speed?
Coordination is needed #1
metadata encapsulation
/**
*
* @param swarmId
* @param seeder true if seeder
* @param complete true if availabilityMap, false if update
* @param blockIds in list of chunks, or availabilityMap
* @constructor
*/
function Have(swarmId, seeder, availabilityMap, blockIds) {
this.tag = exports.P2P_HAVE;
this.swarmId = swarmId;
this.seeder = seeder;
this.blockIds = [];
if (!seeder) {
if (availabilityMap) {
this.complete = true
this.availabilityMap = availabilityMap;
}
else {
this.complete = false
this.blockIds = blockIds;
}
}
} // swarmId (4bytes)
// chunkIds - optional - array of ids each 4bytes
// Example of 2 chunks encoding: 0x010203040A0B0C0D. 0x01-0x04 are the first encoded chunk, 0x0A-0x0D are the second encoded chunk)
function Request(swarmId, chunkIds) {
this.tag = exports.P2P_REQUEST
this.swarmId = swarmId;
if (!chunkIds) chunkIds = [];
this.chunkIds = chunkIds;
}
// attributes (ordered)
// swarmId - 4bytes - only the intial 4 bytes will be encoded
// length: chunk size + swarmId length under 1200 bytes constraint
// constraints:
// peer will send only hash varified chunks
function Data(swarmId, chunkId, payload) {
this.tag = exports.P2P_DATA;
this.swarmId = swarmId;
this.chunkId = chunkId;
this.payload = payload;
}
Coordination is needed #2
(serialization)
var p2p_have_encode = function (message) {
var buffers = [];
var swarmId = ascii2ab(BinaryProtocol.packSwarmId(message.swarmId));
buffers.push(swarmId);
var seeder = BoolToUInt8Array(message.seeder);
buffers.push(seeder);
var complete = BoolToUInt8Array(message.complete);
buffers.push(complete);
if (message.complete) {
buffers.push(message.availabilityMap);
} else {
var blockIdsEncoded = UInt32ToUInt8Array(message.blockIds);
buffers.push(blockIdsEncoded);
}
return BinaryProtocol.concat(buffers);
};
var p2p_have_decode = function (buffer, index, length) {
var blockIds;
var availabilityMap;
var swarmId = ab2ascii(buffer.slice(index, index + BinaryProtocol.transferedSwarmIdSize));
index += BinaryProtocol.transferedSwarmIdSize;
length -= BinaryProtocol.transferedSwarmIdSize;
var seeder = UInt8ArrayToBool(buffer, index);
index++;
length--;
if (!seeder) {
var complete = UInt8ArrayToBool(buffer, index);
index++;
length--;
if (complete) {
availabilityMap = buffer.slice(index, index + length);
} else { //update
blockIds = [];
while (length > 0) {
blockIds.push(UInt8ArrayToInt32(buffer, index));
index += 4;
length -= 4;
}
}
}
return new protocol.Have(swarmId, seeder, availabilityMap, blockIds);
};
WebRTC Multicasting
is even harder
- All in javascript (for good and bad)
- High churn
- Users enter and leave rapidly
- Node.js helps
- Browser compatibility, interop issues
- Some users have unsupported browsers - IE, Safari, Opera
- Some browsers don't like other browsers - Chrome, Firefox
Sharefest
- Easiest file sharing
-
Open source
(join the growing community on github) - Anyone can create a "torrent"
- Uses many HTML5 apis
- File API
- Filesystem API
- IndexDB
- WebSockets
- And... WebRTC DataChannels
Sharefest Architecture
Sharefest Architecture
(mesh network)
Sharefest Lessons Learned
- Unreliable is usable
- Chrome memory management is problematic
- Latency of handshake is long!
- WebRTC has quirks
- SDP hacks
- Over capacity connections throw errors
- Many others were resolved
-
256 connections per user is the hard limit - but your chrome will probably crash sooner
- File size is a limit, but FS API can help if there's some free space at the endpoint
- Binary serialization of chunks and metadata over base64 is heavy
Sharefest numbers
- 91% of the connections are STUN
- Median p2p connection yield 90KB/s
- Local networks usually yield 2MB/s
- Units
- Chunk Size = 800bytes
-
Block Size = 1MB
chrome://webrtc-internals/ is useful
Last Conclusion from Sharefest:
Node.js is your friend
Peer-assisted
(or HTTP-assisted if you are real p2p fan)
Hybrid Approaches
-
Aggressive -- Use P2P and HTTP all the time.
Good for file downloading -
Server-friendly -- Use P2P as much as possible.
Good for video streaming. Requires client side logic - P2P friendly -- Use servers as much as possible
when there's a capacity problem, use P2P.
Good for websites that want to use P2P as an "insurance"
https://github.com/Peer5/P2PXHR
Peer5 XHR-like API
- Data channels is hard:
- learn a new javascript api
- create a signaling protocol
- STUN/TURN
- Big messages
- Really big messages
- Multi-party p2p
- P2PXHR - Empower your xhr's
The API
peer5.Request = function(peer5_options){
peer5.Request.prototype = {
/* -- Attributes -- */
this.readyState;
this.response;
this.status;
/* -- Methods -- */
open: function(method, id) {},
send: function() {},
abort: function(options) {},
/* -- EVENTS --
onreadystatechange: function(e) {},
onloadstart: function(e) {},
onprogress: function(e) {},
onerror:function(e) {},
onload:function(e) {},
}
}
Peer5 XHR-like API
An API compatible with W3C XMLHttpRequest (AJAX):
var xhr = new XMLHttpRequest(); // (taken from html5rocks.com) xhr.open('GET', '/path/to/image.png', true); xhr.responseType = 'blob'; xhr.onload = function(e) { if (this.status == 200) { // Note: .response instead of .responseText var blob = new Blob([this.response], {type: 'image/png'}); ... } }; xhr.send();
With the power of the peers to assist seamlessly (see on peer5.com)
//ctor takes optional parameters to control hybrid downloading var request = new peer5.Request();
request.open('GET', 'http://path/to/a/big/file.json');
request.onload = function(e){
console.log(this.response);
... }; request.send();
File download example
var request = new peer5.Request();
request.open("GET",url);
request.onprogress = function(e){
console.log(e.loadedHTTP);
console.log(e.loadedP2P);
$("#amount-http").text(e.loadedHTTP);
$("#amount-p2p").text(e.loadedP2P);
$("#progress").attr("value",((e.loadedHTTP+e.loadedP2P)/e.total)*100);
};
request.onload = function(e){
$("#progress").hide();
var saveB = document.getElementById("saveBtn");
saveB.hidden=false;
saveB.onclick = function(){save(e.currentTarget.response);};
};
request.onerror = function(e){
console.log(e.error);
}
request.send();
Img download example
var request = new peer5.Request();
request.open("GET",url);
request.onprogress = function(e){
console.log(e.loadedHTTP);
console.log(e.loadedP2P);
};
request.onload = function(e){
console.log(e.currentTarget.response);
$("#cat").attr("src",e.currentTarget.response);
};
request.onerror = function(e){
console.log(e.error);
}
request.send();
Sharefest over api
downloader
peer5Request = new peer5.Request({downloadMode:"p2p"});
peer5Request.onload = function(e){
console.log("Finished downloading the file");
finishProgressUI(e.loaded, e.total);
}
peer5Request.onloadstart = function(e){
initiateProgressUI();
}
peer5Request.onprogress = function(e){
if(this.readyState == 1) return;
updateProgressUI(e.loaded,e.total);
}
peer5Request.onreadystatechange = function(e){
if(this.readyState == 1){
document.getElementById('box-text').textContent = "Initiating...";
}else if(this.readyState == 2){
document.getElementById('box-text').textContent = "";
}
}
peer5Request.onswarmstatechange = function(e){
if(e.numOfPeers){
$('#numOfPeers').html('Connected to ' + e.numOfPeers + ' peers');
$('#numOfPeers').show();
}
}
peer5Request.onerror = function(e){
switch (this.status) {
case peer5.Request.SWARMID_NOT_FOUND_ERR:
roomNotFoundUI();
break;
case peer5.Request.CHROME_ONLY_SWARM_ERR:
chromeOnlyRoomUI();
break;
case peer5.Request.FIREFOX_ONLY_SWARM_ERR:
firefoxOnlyRoomUI();
break;
case peer5.Request.FILE_SIZE_ERR:
fileSizeTooBigUI();
break;
}
}
peer5Request.open("GET",document.location.pathname.substring(1));
peer5Request.send();
Sharefest over api
Uploader
var peer5Request = new peer5.Request({downloadMode:"p2p"});
var file = files[0];
peer5Request.onprogress = function(e){
updateShareProgressUI(e.loaded, e.total);
}
peer5Request.onload = function(e){
userState = {isSeeder:true,origin:true};
var url = this.getFileInfo().swarmId;
history.pushState({}, url, url);
ga('send', 'pageview'); // should notify the current URL
updateList(file.name);
$('#box-text').append(' is ready to be shared!' + '<br>');
showLink();
updateShareButtons();
}
peer5Request.open('POST');
peer5Request.onerror = function(e){
switch (this.status) {
case peer5.Request.FILE_SIZE_ERR:
fileSizeTooBigUI();
break;
}
}
peer5Request.send(file);
Codelab
- Encrypted send
- Clipboard share
- above/below the fold loading
- and more
Thank you
hadar@peer5.com // sharefest.me
Data channels
By Hadar Weiss
Data channels
- 2,891