Fipes: Beating the sneakernet

OSDC.fr (2012-10-13)

@tOkeshu

How do you share files?

The 949 Problem

img

« Every time you email a file to yourself so you can pull it up on your friend's laptop, Tim Berners-Lee sheds a single tear. »

xkcd 949

Sneakers!

img

sneakernet: /snee´ker·net/, n.

Term used (generally with ironic intent) for transfer of electronic information by physically carrying tape, disks, or some other media from one machine to another.

The Jargon File

Features

img

To beat the sneakernet

We need a privacy friendly open source web application that doesn't ask me for an account and doesn't keep any data about me or my files.

Files + Pipes = Fipes

The plan

So you just delete files after a certain time, right?

NO, I don't

I don't need to. Because I don't store any data on the server.

Worflow

img

As soon as the data hits the server, the data is immediatly sent to the downloader. No storage, ever.

Show us the code!

Full JavaScript frontend:

Erlang on the server side:


$('input[type="file"]').on('change', function(event) {
    var files = _.map(event.target.files, function(file) {
        return new App.Models.File({
            obj  : file,
            name : file.name,
            type : file.type,
            size : file.size
        });
    });

    App.Files.add(files).save();
});


var ws = new WebSocket("ws://fipelines.org/fipes/4CAEFAFC39BDE");

this.ws.onmessage = function (evt) {
    var event = tnetstrings.parse(evt.data).value;

    switch (event.type) {
    case "uid":
        App.UID = uid;
        break;
    case "stream":
        // Someone ask for a file
        stream(ws, event);
        break;
    case "file.new":
        // Someone offers a new file.
        if (event.file.owner != App.UID) {
            App.Files.add(event.file);
        }
        break;
    case "file.remove":
        // Someone removed a file.
        if (event.file.owner != App.UID) {
            var file = App.Files.get(event.file.id);
            App.Files.remove(file);
        }
        break;
    }
};

Example of a real time event


{
    type: "file.new",
    file: {
        id: "4CAEFD60B8043",
        owner: "4CAEFCFB7C116",
        name: "lolcat.jpg",
        type: "image/jpeg",
        size: 625230
    }
}


// request = {type: "stream", file: "4CBECEBF793B4", downloader: "4CBECEBB77B6C", seek: 0}
var stream = function(ws, request) {
    var file = App.Files.get(request.file).get('obj'); // File object
    var reader = new FileReader;
    var seek = request.seek;
    var slice = 1024 * 512; // 512 KB

    // Stream the file
    if (seek < file.size) {
        var blob = file.slice(seek, seek + slice);
        reader.readAsBinaryString(blob);
    // Stop the stream
    } else {
        var eos = tnetstrings.dump({
            type: "eos",
            downloader: request.downloader
        });
        ws.send(eos);
    }

    reader.onload = function(e) {
        var data = btoa(e.target.result);
        var chunkEvent = tnetstrings.dump({
            type : "chunk",
            payload : data,
            downloader : request.downloader
        });
        ws.send(chunkEvent);
    }
};

The uploader process


rpc(Fipe, <<"chunk">>, Event) ->
    Payload    = proplists:get_value(payload, Event),
    Uid        = proplists:get_value(downloader, Event),

    [{{Fipe, Uid}, Downloader}] = ets:lookup(downloaders, {Fipe, Uid}),
    Downloader ! {chunk, base64:decode(Payload)};

rpc(Fipe, <<"eos">>, Event) ->
    Uid = proplists:get_value(downloader, Event),
    [{{Fipe, Uid}, Downloader}] = ets:lookup(downloaders, {Fipe, Uid}),
    Downloader ! {chunk, eos};

The downloader process


stream(Fipe, Uid, Req) ->
    receive
        {chunk, eos} ->
            ets:delete(downloaders, {Fipe, Uid}),
            {ok, Req};
        {chunk, Chunk} ->
            cowboy_http_req:chunk(Chunk, Req),
            stream(Fipe, Uid, Req)
    end.

« I lied to you » and the future.

More HTML5:

Questions?

Code: github.com/tOkeshu/fipes (GNU AGPLv3)

Try it online: fipelines.org