WebRTC ブラウザ / iOS / Android とりあえず繋いでみる まとめ

なんとなく落ち着いたのでまとめ。
webRTC で ブラウザ(要クロム) と iOS(アプリ) と Android(アプリ)を繋ぐサンプル。

成果物

iOS

Android

java: https://github.com/nakadoribooks/webrtc-android/releases/tag/v0.0.4
kotlin : https://github.com/nakadoribooks/webrtc-android-kotlin/releases/tag/v0.0.1

ブラウザ

過去記事

iOS/Android はWebRTCライブラリのビルドが必要になります。
簡易wampサーバはこちら → wampサーバーを作って、herokuに置く

iOS

1. iOS用のframeworkのビルド
2. 繋げてみる
3. tricke ICE
4. 複数台接続

Android

1. android用のライブラリをビルド
2. 繋げてみる
3. trickle ICE
4. 複数台接続

ブラウザ(要クロム)

1. 繋げて見る
2. 複数人での接続

流れ

参考

• https://html5experts.jp/mganeko/20112/
• https://html5experts.jp/mganeko/20273/

設計

インタフェース

各プラットフォームをなるだけ揃える。

WebRTC

今回のメイン。メディアストリーム(音声とか動画が乗っかってるやつ)をごにょごにゅするやつ。各種PeerConnectionのラッパー

• iOS : WebRTCのソースコードよりビルド → iOS | WebRTC
• Android : WebRTCのソースコードよりビルド → Android | WebRTC
• ブラウザ : ブラウザ埋め込みのやつ → RTCPeerConnection

iOS

WebRTCInterface.swift

protocol WebRTCInterface {

    static func setup()
    static func disableVideo()
    static func enableVideo()
    static var localStream:RTCMediaStream?{ get }
    
    init(callbacks:WebRTCCallback)
    func createOffer()
    func receiveOffer(sdp:String)
    func receiveAnswer(sdp:String)
    func receiveCandidate(sdp:String, sdpMid:String, sdpMLineIndex:Int32)
    func close()
    
}

// Callback

typealias WebRTCOnCreateOfferHandler = (_ sdp:String) -> ()
typealias WebRTCOnCreateAnswerHandler = (_ sdp:String) -> ()
typealias WebRTCOnIceCandidateHandler = (_ sdp:String, _ sdpMid:String, _ sdpMLineIndex:Int32) -> ()
typealias WebRTCOnAddedStream = (_ stream:RTCMediaStream) -> ()
typealias WebRTCOnRemoveStream = (_ stream:RTCMediaStream) -> ()

typealias WebRTCCallback = (onCreateOffer:WebRTCOnCreateOfferHandler
    , onCreateAnswer:WebRTCOnCreateAnswerHandler
    , onIceCandidate:WebRTCOnIceCandidateHandler
    , onAddedStream:WebRTCOnAddedStream
    , onRemoveStream:WebRTCOnRemoveStream)

Android

WebRTCInterface.java

interface WebRTCInterface {

    void createOffer();
    void receiveOffer(String sdp);
    void receiveAnswer(String sdp);
    void receiveCandidate(String sdp, String sdpMid, int sdpMLineIndex);
    void close();

}

interface WebRTCCallbacks{

    void onCreateOffer(String sdp);
    void onCreateAnswer(String sdp);
    void onAddedStream(MediaStream mediaStream);
    void onIceCandidate(String sdp, String sdpMid, int sdpMLineIndex);

}

ブラウザ

webrtc.js

// interface
createOffer()
receiveAnswer(sdp)
receiveOffer(sdp)
receiveCandidate(candidate)
close()

// callback
{
    onCreateOffer: (sdp) => {}
    , onCreateAnswer: (sdp) => {}
    , onIceCandidate: (candidate) => {}
    , onAddedStream: (stream) => {}
    , onRemoveStream: (stream) =>{}
}

Wamp

外とのやりとり。
プラットフォーム毎の各種wampライブラリのラッパー。
ios : swamp
Android : jawampa
ブラウザ : autobhan

swampって「沼地」っていう意味なんだ。。
ハマりそう。

iOS

WampInterface.swift

protocol WampInterface {
    
    init(roomId:String, userId:String, callbacks:WampCallback)
    func connect()
    func publishCallme()
    func publishOffer(targetId:String ,sdp:String)
    func publishAnswer(targetId:String, sdp:String)
    func publishCandidate(targetId:String, candidate:String)
    
}

// callbacks

typealias WampOnOpenHandler = (()->())
typealias WampReceiveAnswerHandler = ((_ targetId:String, _ sdp:String)->())
typealias WampReceiveOfferHandler = ((_ targetId:String, _ sdp:String)->())
typealias WampReceiveCandidateHandler = ((_ targetId:String, _ sdp:String, _ sdpMid:String, _ sdpMLineIndex:Int32)->())
typealias WampReceiveCallmeHandler = ((_ targetId:String)->())
typealias WampOncloseConnectionHandler = ((_ targetId:String)->())

typealias WampCallback = (onOpen:WampOnOpenHandler
    , onReceiveAnswer:WampReceiveAnswerHandler
    , onReceiveOffer:WampReceiveOfferHandler
    , onReceiveCallme:WampReceiveCallmeHandler
    , onCloseConnection:WampOncloseConnectionHandler
    , onReceiveCandidate:WampReceiveCandidateHandler)

Android

WampInterface.java

interface WampInterface {

    void connect();
    void publishCallme();
    void publishOffer(String targetId, String sdp);
    void publishAnswer(String targetId, String sdp);
    void publishCandidate(String targetId, String candidate);

}

interface WampCallbacks {

    void onOpen();
    void onReceiveAnswer(String targetId, String sdp);
    void onReceiveOffer(String taretId, String sdp);
    void onIceCandidate(String targetId, String sdp, String sdpMid, int sdpMLineIndex);
    void onReceiveCallme(String targetId);
    void onCloseConnection(String targetId);

}

ブラウザ

wamp.js

// interface
connect()
publishCallme()
publishAnswer(targetId, sdp)
publishOffer(targetId, sdp)
publishCandidate(targetId, candidate)
publishClose()

// callback
{
    onOpen: () => { },
    onReceiveAnswer: (targetId, sdp) => {},
    onReceiveOffer: (targetId, sdp) =>{
    onReceiveCandidate: (targetId, candidate) => {},
    onReceiveCallme:(targetId)=>{},
    onCloseConnection:(targetId) => {}
}

Connection

接続一個分の管理

iOS

ConnectionInterface.swift

protocol ConnectionInterface{

    init(myId:String, targetId:String, wamp:WampInterface, onAddedStream:@escaping ConnectionOnAddedStream)
    var targetId:String?{ get }
    func publishOffer()
    func receiveOffer(sdp:String)
    func receiveAnswer(sdp:String)
    func receiveCandidate(sdp:String, sdpMid:String, sdpMLineIndex:Int32)
    func close()
    
}

// callback

typealias ConnectionOnAddedStream = (_ stream:RTCMediaStream)->()

Android

ConnectionInterface.java

interface ConnectionInterface {

    String targetId();
    void publishOffer();
    void receiveOffer(String sdp);
    void receiveAnswer(String sdp);
    void receiveCandidate(String candidate, String sdpMid, int sdpMLineIndex);
    void close();

}

interface ConnectionCallbacks{
    void onAddedStream(MediaStream mediaStream);
}

ブラウザ

connection.js

// interface
get targetId()
publishOffer()
receiveOffer(sdp)
receiveAnswer(answerSdp)
receiveCandidate(candidate)
close()

// callback
{
    onAddedStream:(remoteStream)=>{}
}

実装例

やっぱiOSで。

App (ViewController)

自分のUserId の Topicを監視。

Wamp.callback → App の部分
receiveOffer / receiveCallme のタイミングでConnectionを作る。
receiveAnswer/receiveCandidate では、すでに作成済みのConnectionを探してそれに渡す。

ViewController.swift

private func setupWamp(){
        
    let wamp = Wamp(roomId: roomId, userId: userId
    , callbacks: (
        onOpen:{() -> Void in
            print("onOpen")
            
            self.wamp.publishCallme()
        }
        , onReceiveOffer:{(targetId:String, sdp:String) -> Void in
            print("onReceiveOffer")
            
            let connection = self.createConnection(targetId: targetId)
            connection.receiveOffer(sdp: sdp)
        }
        , onReceiveAnswer:{(targetId:String, sdp:String) -> Void in
            print("onReceiveAnswer")
            
            guard let connection = self.findConnection(targetId: targetId) else{
                return
            }
            
            connection.receiveAnswer(sdp: sdp)
        }
        , onReceiveCandidate:{(targetId:String, sdp:String, sdpMid:String, sdpMLineIndex:Int32) -> Void in
            guard let connection = self.findConnection(targetId: targetId) else{
                return
            }
            
            connection.receiveCandidate(sdp: sdp, sdpMid: sdpMid, sdpMLineIndex: sdpMLineIndex)
        }
        , onReceiveCallme:{(targetId:String) -> Void in
            print("onReceivCallme")
            let connection = self.createConnection(targetId:targetId)
            connection.publishOffer()
        }
        , onCloseConnection:{(targetId:String) -> Void in
            print("onCloseConnection")
            
            // removeConnection
            guard let removeIndex = self.connectionList.index(where: { (row) -> Bool in
                return row.targetId == targetId
            }) else{
                return
            }
            
            let connection = self.connectionList.remove(at: removeIndex)
            connection.close()
            
            // removeView
            guard let streamIndex = self.remoteRenderList.index(where: { (row) -> Bool in
                return row.targetId == targetId
            }) else{
                return;
            }
            
            let remoteRender = self.remoteRenderList.remove(at: streamIndex)
            remoteRender.view.removeFromSuperview()
            
            self.calcRemoteViewPosition()
        }
   ))
    
    self.wamp = wamp
}

ストリームゲットしたらView作って表示

Connection.callback → App の部分

ViewController.swift

private func createConnection(targetId:String)->Connection{
    let connection = Connection(myId: userId, targetId: targetId, wamp: wamp) { (remoteStream) in        
        let remoteRenderView = RemoteRenderView(stream: remoteStream, targetId: targetId)
        self.remoteLayer.addSubview(remoteRenderView.view)

        self.calcRemoteViewPosition()
    }
}

Connection

シグナリング

WebRTC.callback → Connection → (→Wamp→相手)の部分

Connection.swift

webRtc = WebRTC(callbacks: (
    onCreateOffer: {(sdp:String) -> Void in
        self.wamp.publishOffer(targetId: targetId, sdp: sdp)
    }
    , onCreateAnswer: {(sdp:String) -> Void in
        self.wamp.publishAnswer(targetId: targetId, sdp: sdp)
    }
    , onIceCandidate: {(sdp:String, sdpMid:String, sdpMLineIndex:Int32) -> Void in
        
        let dic:NSDictionary = [
            "type": "candidate"
            , "sdpMid": sdpMid
            , "sdpMLineIndex": sdpMLineIndex
            , "candidate": sdp
        ]
        
        do{
            let jsonData = try JSONSerialization.data(withJSONObject: dic, options: [])
            let jsonStr = String(bytes: jsonData, encoding: .utf8)!
            self.wamp.publishCandidate(targetId: targetId, candidate: jsonStr)
        }catch let e{
            print(e)
        }
        
    }
))

WebRTC

peerconnectionに対しての操作。
offerを送る側と、offerをもらう側。

offerを送る側のとき

createOffer → (wamp → 相手 → wamp) → receiveAnswer → \(o-o)/

WebRTC.swift

func createOffer(){
        
    // 1. offerを作る
    peerConnection?.offer(for: WebRTCUtil.mediaStreamConstraints(), completionHandler: { (description, error) in
        
        guard let description = description else{
            print("----- no description ----")
            return;
        }
        
        // 2.ローカルにSDPを登録
        self.peerConnection?.setLocalDescription(description, completionHandler: { (error) in
            // 3. offer を送る
            guard let description = self.peerConnection?.localDescription else{
                return;
            }
            
            let dic:NSDictionary = [
                "type": RTCSessionDescription.string(for: description.type)
                , "sdp": description.sdp
            ]
            
            do{
                let jsonData = try JSONSerialization.data(withJSONObject: dic, options: [])
                let sdp = String(bytes: jsonData, encoding: .utf8)!
                self.callbacks.onCreateOffer(sdp)
            }catch let e{
                print(e)
                return
            }
            
        })
    })
}

func receiveAnswer(sdp:String){
    let sdp = RTCSessionDescription(type: .answer, sdp: sdp)    
    peerConnection?.setRemoteDescription(sdp, completionHandler: { (error) in

    })
}

offerをもらう側のとき

(相手 → wamp) → receiveOffer → onCreatedAnswer → (→ wamp → 相手) → \(o-o)/

WebRTC.swift

func receiveOffer(sdp:String){
    // 1. remote SDP を登録
    let remoteSdp = RTCSessionDescription(type: .offer, sdp: sdp)
    peerConnection?.setRemoteDescription(remoteSdp, completionHandler: { (error) in

        // 2. answerを作る
        self.peerConnection?.answer(for: WebRTCUtil.answerConstraints(), completionHandler: { (sdp, error) in
            
            guard let sdp = sdp else{
                print("can not create sdp")
                return;
            }
            
            // 3.ローカルにSDPを登録
            self.peerConnection?.setLocalDescription(sdp, completionHandler: { (error) in
                
                // 3. answer を送る
                guard let description = self.peerConnection?.localDescription else{
                    return;
                }
                
                let dic:NSDictionary = [
                    "type": RTCSessionDescription.string(for: description.type)
                    , "sdp": description.sdp
                ]
                
                do{
                    let jsonData = try JSONSerialization.data(withJSONObject: dic, options: [])
                    let sdp = String(bytes: jsonData, encoding: .utf8)!
                    self.callbacks.onCreateAnswer(sdp)
                }catch let e{
                    print(e)
                    return
                }                    
            })
            
        })
    })
}

終わり。

次

まとめといてなんですが、自分で実装する必要がなければ素直にありものを使うのがおすすめです。いろいろサービスが出ているようです。

• [Tokbox] (https://tokbox.com/)
• SkyWay
• twillio