497 lines
14 KiB
JavaScript
497 lines
14 KiB
JavaScript
/*
|
|
* Copyright (C) 2019 The Android Open Source Project
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
function createDataChannel(pc, label, onMessage) {
|
|
console.debug('creating data channel: ' + label);
|
|
let dataChannel = pc.createDataChannel(label);
|
|
// Return an object with a send function like that of the dataChannel, but
|
|
// that only actually sends over the data channel once it has connected.
|
|
return {
|
|
channelPromise: new Promise((resolve, reject) => {
|
|
dataChannel.onopen = (event) => {
|
|
resolve(dataChannel);
|
|
};
|
|
dataChannel.onclose = () => {
|
|
console.debug(
|
|
'Data channel=' + label + ' state=' + dataChannel.readyState);
|
|
};
|
|
dataChannel.onmessage = onMessage ? onMessage : (msg) => {
|
|
console.debug('Data channel=' + label + ' data="' + msg.data + '"');
|
|
};
|
|
dataChannel.onerror = err => {
|
|
reject(err);
|
|
};
|
|
}),
|
|
send: function(msg) {
|
|
this.channelPromise = this.channelPromise.then(channel => {
|
|
channel.send(msg);
|
|
return channel;
|
|
})
|
|
},
|
|
};
|
|
}
|
|
|
|
function awaitDataChannel(pc, label, onMessage) {
|
|
console.debug('expecting data channel: ' + label);
|
|
// Return an object with a send function like that of the dataChannel, but
|
|
// that only actually sends over the data channel once it has connected.
|
|
return {
|
|
channelPromise: new Promise((resolve, reject) => {
|
|
let prev_ondatachannel = pc.ondatachannel;
|
|
pc.ondatachannel = ev => {
|
|
let dataChannel = ev.channel;
|
|
if (dataChannel.label == label) {
|
|
dataChannel.onopen = (event) => {
|
|
resolve(dataChannel);
|
|
};
|
|
dataChannel.onclose = () => {
|
|
console.debug(
|
|
'Data channel=' + label + ' state=' + dataChannel.readyState);
|
|
};
|
|
dataChannel.onmessage = onMessage ? onMessage : (msg) => {
|
|
console.debug('Data channel=' + label + ' data="' + msg.data + '"');
|
|
};
|
|
dataChannel.onerror = err => {
|
|
reject(err);
|
|
};
|
|
} else if (prev_ondatachannel) {
|
|
prev_ondatachannel(ev);
|
|
}
|
|
};
|
|
}),
|
|
send: function(msg) {
|
|
this.channelPromise = this.channelPromise.then(channel => {
|
|
channel.send(msg);
|
|
return channel;
|
|
})
|
|
},
|
|
};
|
|
}
|
|
|
|
class DeviceConnection {
|
|
#pc;
|
|
#control;
|
|
#description;
|
|
|
|
#cameraDataChannel;
|
|
#cameraInputQueue;
|
|
#controlChannel;
|
|
#inputChannel;
|
|
#adbChannel;
|
|
#bluetoothChannel;
|
|
|
|
#streams;
|
|
#streamPromiseResolvers;
|
|
#micSenders = [];
|
|
#cameraSenders = [];
|
|
#camera_res_x;
|
|
#camera_res_y;
|
|
|
|
#onAdbMessage;
|
|
#onControlMessage;
|
|
#onBluetoothMessage;
|
|
|
|
constructor(pc, control) {
|
|
this.#pc = pc;
|
|
this.#control = control;
|
|
this.#cameraDataChannel = pc.createDataChannel('camera-data-channel');
|
|
this.#cameraDataChannel.binaryType = 'arraybuffer';
|
|
this.#cameraInputQueue = new Array();
|
|
var self = this;
|
|
this.#cameraDataChannel.onbufferedamountlow = () => {
|
|
if (self.#cameraInputQueue.length > 0) {
|
|
self.sendCameraData(self.#cameraInputQueue.shift());
|
|
}
|
|
};
|
|
this.#inputChannel = createDataChannel(pc, 'input-channel');
|
|
this.#adbChannel = createDataChannel(pc, 'adb-channel', (msg) => {
|
|
if (this.#onAdbMessage) {
|
|
this.#onAdbMessage(msg.data);
|
|
} else {
|
|
console.error('Received unexpected ADB message');
|
|
}
|
|
});
|
|
this.#controlChannel = awaitDataChannel(pc, 'device-control', (msg) => {
|
|
if (this.#onControlMessage) {
|
|
this.#onControlMessage(msg);
|
|
} else {
|
|
console.error('Received unexpected Control message');
|
|
}
|
|
});
|
|
this.#bluetoothChannel =
|
|
createDataChannel(pc, 'bluetooth-channel', (msg) => {
|
|
if (this.#onBluetoothMessage) {
|
|
this.#onBluetoothMessage(msg.data);
|
|
} else {
|
|
console.error('Received unexpected Bluetooth message');
|
|
}
|
|
});
|
|
this.#streams = {};
|
|
this.#streamPromiseResolvers = {};
|
|
|
|
pc.addEventListener('track', e => {
|
|
console.debug('Got remote stream: ', e);
|
|
for (const stream of e.streams) {
|
|
this.#streams[stream.id] = stream;
|
|
if (this.#streamPromiseResolvers[stream.id]) {
|
|
for (let resolver of this.#streamPromiseResolvers[stream.id]) {
|
|
resolver();
|
|
}
|
|
delete this.#streamPromiseResolvers[stream.id];
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
set description(desc) {
|
|
this.#description = desc;
|
|
}
|
|
|
|
get description() {
|
|
return this.#description;
|
|
}
|
|
|
|
get imageCapture() {
|
|
if (this.#cameraSenders && this.#cameraSenders.length > 0) {
|
|
let track = this.#cameraSenders[0].track;
|
|
return new ImageCapture(track);
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
get cameraWidth() {
|
|
return this.#camera_res_x;
|
|
}
|
|
|
|
get cameraHeight() {
|
|
return this.#camera_res_y;
|
|
}
|
|
|
|
get cameraEnabled() {
|
|
return this.#cameraSenders && this.#cameraSenders.length > 0;
|
|
}
|
|
|
|
getStream(stream_id) {
|
|
return new Promise((resolve, reject) => {
|
|
if (this.#streams[stream_id]) {
|
|
resolve(this.#streams[stream_id]);
|
|
} else {
|
|
if (!this.#streamPromiseResolvers[stream_id]) {
|
|
this.#streamPromiseResolvers[stream_id] = [];
|
|
}
|
|
this.#streamPromiseResolvers[stream_id].push(resolve);
|
|
}
|
|
});
|
|
}
|
|
|
|
#sendJsonInput(evt) {
|
|
this.#inputChannel.send(JSON.stringify(evt));
|
|
}
|
|
|
|
sendMousePosition({x, y, down, display_label}) {
|
|
this.#sendJsonInput({
|
|
type: 'mouse',
|
|
down: down ? 1 : 0,
|
|
x,
|
|
y,
|
|
display_label,
|
|
});
|
|
}
|
|
|
|
// TODO (b/124121375): This should probably be an array of pointer events and
|
|
// have different properties.
|
|
sendMultiTouch({idArr, xArr, yArr, down, slotArr, display_label}) {
|
|
this.#sendJsonInput({
|
|
type: 'multi-touch',
|
|
id: idArr,
|
|
x: xArr,
|
|
y: yArr,
|
|
down: down ? 1 : 0,
|
|
slot: slotArr,
|
|
display_label: display_label,
|
|
});
|
|
}
|
|
|
|
sendKeyEvent(code, type) {
|
|
this.#sendJsonInput({type: 'keyboard', keycode: code, event_type: type});
|
|
}
|
|
|
|
disconnect() {
|
|
this.#pc.close();
|
|
}
|
|
|
|
// Sends binary data directly to the in-device adb daemon (skipping the host)
|
|
sendAdbMessage(msg) {
|
|
this.#adbChannel.send(msg);
|
|
}
|
|
|
|
// Provide a callback to receive data from the in-device adb daemon
|
|
onAdbMessage(cb) {
|
|
this.#onAdbMessage = cb;
|
|
}
|
|
|
|
// Send control commands to the device
|
|
sendControlMessage(msg) {
|
|
this.#controlChannel.send(msg);
|
|
}
|
|
|
|
async #useDevice(in_use, senders_arr, device_opt) {
|
|
// An empty array means no tracks are currently in use
|
|
if (senders_arr.length > 0 === !!in_use) {
|
|
console.warn('Device is already ' + (in_use ? '' : 'not ') + 'in use');
|
|
return in_use;
|
|
}
|
|
let renegotiation_needed = false;
|
|
if (in_use) {
|
|
try {
|
|
let stream = await navigator.mediaDevices.getUserMedia(device_opt);
|
|
stream.getTracks().forEach(track => {
|
|
console.info(`Using ${track.kind} device: ${track.label}`);
|
|
senders_arr.push(this.#pc.addTrack(track));
|
|
renegotiation_needed = true;
|
|
});
|
|
} catch (e) {
|
|
console.error('Failed to add stream to peer connection: ', e);
|
|
// Don't return yet, if there were errors some tracks may have been
|
|
// added so the connection should be renegotiated again.
|
|
}
|
|
} else {
|
|
for (const sender of senders_arr) {
|
|
console.info(
|
|
`Removing ${sender.track.kind} device: ${sender.track.label}`);
|
|
let track = sender.track;
|
|
track.stop();
|
|
this.#pc.removeTrack(sender);
|
|
renegotiation_needed = true;
|
|
}
|
|
// Empty the array passed by reference, just assigning [] won't do that.
|
|
senders_arr.length = 0;
|
|
}
|
|
if (renegotiation_needed) {
|
|
this.#control.renegotiateConnection();
|
|
}
|
|
// Return the new state
|
|
return senders_arr.length > 0;
|
|
}
|
|
|
|
async useMic(in_use) {
|
|
return this.#useDevice(in_use, this.#micSenders, {audio: true, video: false});
|
|
}
|
|
|
|
async useCamera(in_use) {
|
|
return this.#useDevice(in_use, this.#micSenders, {audio: false, video: true});
|
|
}
|
|
|
|
sendCameraResolution(stream) {
|
|
const cameraTracks = stream.getVideoTracks();
|
|
if (cameraTracks.length > 0) {
|
|
const settings = cameraTracks[0].getSettings();
|
|
this.#camera_res_x = settings.width;
|
|
this.#camera_res_y = settings.height;
|
|
this.sendControlMessage(JSON.stringify({
|
|
command: 'camera_settings',
|
|
width: settings.width,
|
|
height: settings.height,
|
|
frame_rate: settings.frameRate,
|
|
facing: settings.facingMode
|
|
}));
|
|
}
|
|
}
|
|
|
|
sendOrQueueCameraData(data) {
|
|
if (this.#cameraDataChannel.bufferedAmount > 0 ||
|
|
this.#cameraInputQueue.length > 0) {
|
|
this.#cameraInputQueue.push(data);
|
|
} else {
|
|
this.sendCameraData(data);
|
|
}
|
|
}
|
|
|
|
sendCameraData(data) {
|
|
const MAX_SIZE = 65535;
|
|
const END_MARKER = 'EOF';
|
|
for (let i = 0; i < data.byteLength; i += MAX_SIZE) {
|
|
// range is clamped to the valid index range
|
|
this.#cameraDataChannel.send(data.slice(i, i + MAX_SIZE));
|
|
}
|
|
this.#cameraDataChannel.send(END_MARKER);
|
|
}
|
|
|
|
// Provide a callback to receive control-related comms from the device
|
|
onControlMessage(cb) {
|
|
this.#onControlMessage = cb;
|
|
}
|
|
|
|
sendBluetoothMessage(msg) {
|
|
this.#bluetoothChannel.send(msg);
|
|
}
|
|
|
|
onBluetoothMessage(cb) {
|
|
this.#onBluetoothMessage = cb;
|
|
}
|
|
|
|
// Provide a callback to receive connectionstatechange states.
|
|
onConnectionStateChange(cb) {
|
|
this.#pc.addEventListener(
|
|
'connectionstatechange', evt => cb(this.#pc.connectionState));
|
|
}
|
|
}
|
|
|
|
class Controller {
|
|
#pc;
|
|
#serverConnector;
|
|
|
|
constructor(serverConnector) {
|
|
this.#serverConnector = serverConnector;
|
|
serverConnector.onDeviceMsg(msg => this.#onDeviceMessage(msg));
|
|
}
|
|
|
|
#onDeviceMessage(message) {
|
|
let type = message.type;
|
|
switch (type) {
|
|
case 'offer':
|
|
this.#onOffer({type: 'offer', sdp: message.sdp});
|
|
break;
|
|
case 'answer':
|
|
this.#onAnswer({type: 'answer', sdp: message.sdp});
|
|
break;
|
|
case 'ice-candidate':
|
|
this.#onIceCandidate(new RTCIceCandidate({
|
|
sdpMid: message.mid,
|
|
sdpMLineIndex: message.mLineIndex,
|
|
candidate: message.candidate
|
|
}));
|
|
break;
|
|
case 'error':
|
|
console.error('Device responded with error message: ', message.error);
|
|
break;
|
|
default:
|
|
console.error('Unrecognized message type from device: ', type);
|
|
}
|
|
}
|
|
|
|
async #sendClientDescription(desc) {
|
|
console.debug('sendClientDescription');
|
|
return this.#serverConnector.sendToDevice({type: 'answer', sdp: desc.sdp});
|
|
}
|
|
|
|
async #sendIceCandidate(candidate) {
|
|
console.debug('sendIceCandidate');
|
|
return this.#serverConnector.sendToDevice({type: 'ice-candidate', candidate});
|
|
}
|
|
|
|
async #onOffer(desc) {
|
|
console.debug('Remote description (offer): ', desc);
|
|
try {
|
|
await this.#pc.setRemoteDescription(desc);
|
|
let answer = await this.#pc.createAnswer();
|
|
console.debug('Answer: ', answer);
|
|
await this.#pc.setLocalDescription(answer);
|
|
await this.#sendClientDescription(answer);
|
|
} catch (e) {
|
|
console.error('Error processing remote description (offer)', e)
|
|
throw e;
|
|
}
|
|
}
|
|
|
|
async #onAnswer(answer) {
|
|
console.debug('Remote description (answer): ', answer);
|
|
try {
|
|
await this.#pc.setRemoteDescription(answer);
|
|
} catch (e) {
|
|
console.error('Error processing remote description (answer)', e)
|
|
throw e;
|
|
}
|
|
}
|
|
|
|
#onIceCandidate(iceCandidate) {
|
|
console.debug(`Remote ICE Candidate: `, iceCandidate);
|
|
this.#pc.addIceCandidate(iceCandidate);
|
|
}
|
|
|
|
ConnectDevice(pc) {
|
|
this.#pc = pc;
|
|
console.debug('ConnectDevice');
|
|
// ICE candidates will be generated when we add the offer. Adding it here
|
|
// instead of in _onOffer because this function is called once per peer
|
|
// connection, while _onOffer may be called more than once due to
|
|
// renegotiations.
|
|
this.#pc.addEventListener('icecandidate', evt => {
|
|
if (evt.candidate) this.#sendIceCandidate(evt.candidate);
|
|
});
|
|
this.#serverConnector.sendToDevice({type: 'request-offer'});
|
|
}
|
|
|
|
async renegotiateConnection() {
|
|
console.debug('Re-negotiating connection');
|
|
let offer = await this.#pc.createOffer();
|
|
console.debug('Local description (offer): ', offer);
|
|
await this.#pc.setLocalDescription(offer);
|
|
this.#serverConnector.sendToDevice({type: 'offer', sdp: offer.sdp});
|
|
}
|
|
}
|
|
|
|
function createPeerConnection(infra_config) {
|
|
let pc_config = {iceServers: []};
|
|
for (const stun of infra_config.ice_servers) {
|
|
pc_config.iceServers.push({urls: 'stun:' + stun});
|
|
}
|
|
let pc = new RTCPeerConnection(pc_config);
|
|
|
|
pc.addEventListener('icecandidate', evt => {
|
|
console.debug('Local ICE Candidate: ', evt.candidate);
|
|
});
|
|
pc.addEventListener('iceconnectionstatechange', evt => {
|
|
console.debug(`ICE State Change: ${pc.iceConnectionState}`);
|
|
});
|
|
pc.addEventListener(
|
|
'connectionstatechange',
|
|
evt => console.debug(
|
|
`WebRTC Connection State Change: ${pc.connectionState}`));
|
|
return pc;
|
|
}
|
|
|
|
export async function Connect(deviceId, serverConnector) {
|
|
let requestRet = await serverConnector.requestDevice(deviceId);
|
|
let deviceInfo = requestRet.deviceInfo;
|
|
let infraConfig = requestRet.infraConfig;
|
|
console.debug('Device available:');
|
|
console.debug(deviceInfo);
|
|
let pc_config = {iceServers: []};
|
|
if (infraConfig.ice_servers && infraConfig.ice_servers.length > 0) {
|
|
for (const server of infraConfig.ice_servers) {
|
|
pc_config.iceServers.push(server);
|
|
}
|
|
}
|
|
let pc = createPeerConnection(infraConfig);
|
|
|
|
let control = new Controller(serverConnector);
|
|
let deviceConnection = new DeviceConnection(pc, control);
|
|
deviceConnection.description = deviceInfo;
|
|
|
|
return new Promise((resolve, reject) => {
|
|
pc.addEventListener('connectionstatechange', evt => {
|
|
let state = pc.connectionState;
|
|
if (state == 'connected') {
|
|
resolve(deviceConnection);
|
|
} else if (state == 'failed') {
|
|
reject(evt);
|
|
}
|
|
});
|
|
control.ConnectDevice(pc);
|
|
});
|
|
}
|