// JumPHP Screen: a computer's screen on a phone, like VNC but WebRTC (the JumPHP IDE's "Screen on a phone"; contract: JumPHP docs/ide-screen.md).// 1. pairing (the IDE's link: #r=<room>&k=<key>) -> an end-to-end encrypted channel through the relay (channel.js, ciphertext only)// 2. hello -> the PC's user allows this device -> the PC's WebRTC peer sends an offer: H.264 video of its screen + an "input" data channel// 3. the video plays with no jitter buffer; touches become mouse / keys on the PC (touchpad or direct touch, pinch zoom, keyboard)import { Channel } from './channel.js';import { t, applyTexts } from './texts.js';const RELAY = 'wss://jumphp.net/ide-relay/ws';const ICE = [{ urls: 'stun:stun.l.google.com:19302' }];const $ = (id) => document.getElementById(id);const store = { get: (k, d = null) => { try { return localStorage.getItem('screen.' + k) ?? d; } catch { return d; } }, set: (k, v) => { try { v === null ? localStorage.removeItem('screen.' + k) : localStorage.setItem('screen.' + k, v); } catch { /* private mode */ } } };const clamp = (v, lo, hi) => Math.max(lo, Math.min(hi, v));let pairing = null; let channel = null; let pc = null; let input = null; let stopped = true; let attempt = 0;let screenW = 1920; let screenH = 1080; // the PC's screen (from the "size" message)let mode = store.get('mode', 'touchpad'); // touchpad | directlet cur = { x: 0.5, y: 0.5 }; // the PC's cursor, 0..1 (what we sent last)let view = { z: 1, tx: 0, ty: 0, f: 1 }; // zoom, pan, base fit scalelet timers = [];// --- UI states -----------------------------------------------------------------------------------------------------------function gate(state, text = '') { const on = state !== 'live'; $('gate').hidden = !on; $('stage').hidden = on; $('pairForm').hidden = state !== 'pair'; $('wait').hidden = state === 'pair' || !on; $('waitText').textContent = text; $('retry').hidden = state !== 'denied'; document.querySelector('.spinner').hidden = state === 'denied';}function toast(text, ms = 2200) { const el = $('toast'); el.textContent = text; el.hidden = false; clearTimeout(toast.timer); toast.timer = setTimeout(() => { el.hidden = true; }, ms);}// --- connection ------------------------------------------------------------------------------------------------------------async function start() { stopped = false; gate('wait', t('connecting')); try { const ch = await Channel.open({ role: 'remote', key: pairing.key, transport: Channel.relayTransport({ url: window.SCREEN_RELAY || RELAY, room: pairing.room, role: 'remote' }) }); channel = ch; let hello = false; const sayHello = () => { if (!hello) { hello = true; ch.send({ t: 'hello', v: 1, app: 'screen', name: $('deviceName').value.trim() || 'Phone' }); gate('wait', t('waiting')); } }; ch.on('open', sayHello); if (ch.isOpen) sayHello(); ch.on('message', onMessage); ch.on('close', () => { if (!stopped && !pc) gate('wait', t('reconnecting')); }); timers.push(setInterval(() => ch.send({ t: 'ping' }).catch(() => {}), 20000)); } catch (e) { console.warn('[screen] connect', e); retryLater(); }}function retryLater() { if (stopped) return; stop(false); stopped = false; gate('wait', t('reconnecting')); setTimeout(() => { if (!stopped) start(); }, Math.min(15000, 1000 * 2 ** attempt++));}function stop(bye = true) { stopped = true; if (bye) channel?.send({ t: 'bye' }).catch(() => {}); timers.forEach(clearInterval); timers = []; try { pc?.close(); } catch { /* closed */ } pc = null; input = null; const ch = channel; channel = null; setTimeout(() => ch?.close(), bye ? 300 : 0);}async function onMessage(m) { if (!m || typeof m !== 'object') return; switch (m.t) { case 'welcome': attempt = 0; gate('wait', t('starting')); break; case 'info': if (m.width) { screenW = m.width; screenH = m.height; } break; case 'offer': await answer(m.sdp); break; case 'ice': if (pc && m.candidate) pc.addIceCandidate({ candidate: m.candidate, sdpMLineIndex: m.mline | 0 }).catch((e) => console.warn('[screen] ice', e)); break; case 'denied': stop(false); store.set('pairing', null); gate('denied', t('denied_' + (m.reason || 'refused'))); break; case 'error': gate('denied', m.message || 'error'); break; default: break; }}async function answer(sdp) { try { pc?.close(); } catch { /* closed */ } pc = new RTCPeerConnection({ iceServers: ICE, bundlePolicy: 'max-bundle' }); const thisPc = pc; pc.onicecandidate = (e) => { if (e.candidate && e.candidate.candidate) channel?.send({ t: 'ice', candidate: e.candidate.candidate, mline: e.candidate.sdpMLineIndex ?? 0 }); }; pc.ontrack = (e) => { try { e.receiver.jitterBufferTarget = 0; } catch { /* older browsers */ } try { e.receiver.playoutDelayHint = 0; } catch { /* not Chrome */ } const video = $('video'); video.srcObject = e.streams[0] || new MediaStream([e.track]); video.play().catch(() => {}); }; pc.ondatachannel = (e) => { if (e.channel.label === 'input') wireInput(e.channel); }; pc.onconnectionstatechange = () => { if (pc !== thisPc) return; if (pc.connectionState === 'connected') { gate('live'); layout(true); } if (pc.connectionState === 'failed') { toast(t('lost')); channel?.send({ t: 'restart' }); } }; await pc.setRemoteDescription({ type: 'offer', sdp }); const ans = await pc.createAnswer(); await pc.setLocalDescription(ans); channel?.send({ t: 'answer', sdp: pc.localDescription.sdp });}function wireInput(dc) { input = dc; dc.onmessage = (e) => { let m; try { m = JSON.parse(e.data); } catch { return; } if (m.t === 'size') { screenW = m.w; screenH = m.h; stats.encoder = m.encoder || ''; } else if (m.t === 'pong') stats.rtt = Math.round(performance.now() - m.at); }; dc.onopen = () => { send({ t: 'kbps', kbps: Number($('quality').value) }); }; timers.push(setInterval(() => send({ t: 'ping', at: performance.now() }), 2000));}function send(m) { if (input?.readyState === 'open') input.send(JSON.stringify(m)); }// --- view: fit, zoom, pan ----------------------------------------------------------------------------------------------------function videoSize() { const v = $('video'); return [v.videoWidth || screenW, v.videoHeight || screenH]; }function layout(fit = false) { const [vw, vh] = videoSize(); const W = innerWidth; const H = innerHeight; view.f = Math.min(W / vw, H / vh); const el = $('view'); el.style.width = vw * view.f + 'px'; el.style.height = vh * view.f + 'px'; if (fit) { view.z = 1; } apply();}function apply() { const [vw, vh] = videoSize(); const W = innerWidth; const H = innerHeight; view.z = clamp(view.z, 1, 8); const cw = vw * view.f * view.z; const ch = vh * view.f * view.z; view.tx = cw <= W ? (W - cw) / 2 : clamp(view.tx, W - cw, 0); view.ty = ch <= H ? (H - ch) / 2 : clamp(view.ty, H - ch, 0); $('view').style.transform = `translate(${view.tx}px, ${view.ty}px) scale(${view.z})`; drawCursor();}function toScreen(px, py) { // a stage point -> 0..1 of the PC's screen const [vw, vh] = videoSize(); return { x: clamp((px - view.tx) / (vw * view.f * view.z), 0, 1), y: clamp((py - view.ty) / (vh * view.f * view.z), 0, 1) };}function toStage(nx, ny) { const [vw, vh] = videoSize(); return { x: view.tx + nx * vw * view.f * view.z, y: view.ty + ny * vh * view.f * view.z };}function zoomAt(z, px, py) { // keeps the stage point (px, py) on the same screen point const p = toScreen(px, py); view.z = clamp(z, 1, 8); const [vw, vh] = videoSize(); view.tx = px - p.x * vw * view.f * view.z; view.ty = py - p.y * vh * view.f * view.z; apply();}function follow() { // touchpad: the view keeps the cursor in sight when zoomed if (view.z <= 1.01) return; const p = toStage(cur.x, cur.y); const mx = innerWidth * 0.15; const my = innerHeight * 0.15; if (p.x < mx) view.tx += mx - p.x; else if (p.x > innerWidth - mx) view.tx -= p.x - (innerWidth - mx); if (p.y < my) view.ty += my - p.y; else if (p.y > innerHeight - my) view.ty -= p.y - (innerHeight - my); apply();}function drawCursor() { const c = $('cursor'); c.hidden = mode !== 'touchpad'; const p = toStage(cur.x, cur.y); c.style.left = (p.x - view.tx) / view.z + 'px'; c.style.top = (p.y - view.ty) / view.z + 'px'; c.style.transform = `scale(${1 / view.z})`;}// --- mouse moves, coalesced to one per frame ----------------------------------------------------------------------------------let movePending = false;function moveTo(x, y) { cur = { x: clamp(x, 0, 1), y: clamp(y, 0, 1) }; drawCursor(); if (movePending) return; movePending = true; requestAnimationFrame(() => { movePending = false; send({ t: 'm', x: cur.x, y: cur.y }); });}const mods = new Map(); // keysym -> 'on' | 'lock'function releaseMods() { for (const [k, s] of [...mods]) if (s === 'on') { send({ t: 'k', key: k, d: 0 }); mods.delete(k); document.querySelector(`[data-mod="${k}"]`)?.classList.remove('on'); }}function click(b = 1) { send({ t: 'm', x: cur.x, y: cur.y }); send({ t: 'b', b, d: 1 }); send({ t: 'b', b, d: 0 }); navigator.vibrate?.(8); releaseMods();}function key(k) { send({ t: 'k', key: k, d: 1 }); send({ t: 'k', key: k, d: 0 }); releaseMods(); }// --- touch gestures -------------------------------------------------------------------------------------------------------const pts = new Map(); let g = null;const dist = (a, b) => Math.hypot(a.x - b.x, a.y - b.y);const mid = (a, b) => ({ x: (a.x + b.x) / 2, y: (a.y + b.y) / 2 });function down(e) { if (e.pointerType === 'mouse') return mouseDown(e); $('stage').setPointerCapture?.(e.pointerId); pts.set(e.pointerId, { x: e.clientX, y: e.clientY }); if (pts.size === 1) { g = { n: 1, t0: performance.now(), x0: e.clientX, y0: e.clientY, last: { x: e.clientX, y: e.clientY }, moved: false, drag: false, done: false }; g.long = setTimeout(() => { if (!g || g.moved || g.n !== 1) return; navigator.vibrate?.(25); if (mode === 'touchpad') { g.drag = true; send({ t: 'b', b: 1, d: 1 }); } else { const p = toScreen(g.x0, g.y0); moveTo(p.x, p.y); click(3); g.done = true; } }, 450); } else if (pts.size === 2 && g && !g.drag) { clearTimeout(g.long); const [a, b] = [...pts.values()]; g = { n: 2, t0: g.t0, d0: dist(a, b), m0: mid(a, b), last: mid(a, b), z0: view.z, kind: null, scroll: 0, moved: false }; } else if (g) { g.n = Math.max(g.n, pts.size); }}function move(e) { if (e.pointerType === 'mouse') return mouseMove(e); if (!pts.has(e.pointerId) || !g) return; pts.set(e.pointerId, { x: e.clientX, y: e.clientY }); if (g.n === 1 && pts.size === 1) { const dx = e.clientX - g.last.x; const dy = e.clientY - g.last.y; g.last = { x: e.clientX, y: e.clientY }; if (!g.moved && Math.hypot(e.clientX - g.x0, e.clientY - g.y0) > 8) { g.moved = true; if (!g.drag) clearTimeout(g.long); } if (!g.moved && !g.drag) return; if (mode === 'touchpad') { const [vw, vh] = videoSize(); const speed = 1.4 * (1 + Math.min(2, Math.hypot(dx, dy) / 12)); moveTo(cur.x + dx * speed / (vw * view.f * view.z), cur.y + dy * speed / (vh * view.f * view.z)); follow(); } else { const p = toScreen(e.clientX, e.clientY); if (!g.drag) { const s = toScreen(g.x0, g.y0); moveTo(s.x, s.y); send({ t: 'm', x: s.x, y: s.y }); send({ t: 'b', b: 1, d: 1 }); g.drag = true; } moveTo(p.x, p.y); } } else if (g.n === 2 && pts.size === 2) { const [a, b] = [...pts.values()]; const m = mid(a, b); const d = dist(a, b); if (!g.kind && (Math.abs(d - g.d0) > 14 || dist(m, g.m0) > 14)) { g.moved = true; g.kind = mode === 'direct' || Math.abs(d - g.d0) > dist(m, g.m0) ? 'pinch' : 'scroll'; } if (g.kind === 'pinch') { zoomAt(g.z0 * d / g.d0, m.x, m.y); view.tx += m.x - g.last.x; view.ty += m.y - g.last.y; apply(); } else if (g.kind === 'scroll') { g.scroll += m.y - g.last.y; while (Math.abs(g.scroll) >= 28) { const s = Math.sign(g.scroll); send({ t: 'w', dy: -s }); g.scroll -= s * 28; } } g.last = m; }}function up(e) { if (e.pointerType === 'mouse') return mouseUp(e); if (!pts.has(e.pointerId)) return; pts.delete(e.pointerId); if (!g || pts.size) return; clearTimeout(g.long); const quick = performance.now() - g.t0 < 300; if (g.drag) { send({ t: 'b', b: 1, d: 0 }); releaseMods(); } else if (!g.moved && !g.done && quick) { if (g.n === 1) { if (mode === 'direct') { const p = toScreen(g.x0, g.y0); moveTo(p.x, p.y); } click(1); } else if (g.n === 2) click(3); else if (g.n >= 3) click(2); } g = null;}// a mouse (a computer's browser, or a phone with a mouse): directfunction mouseDown(e) { const p = toScreen(e.clientX, e.clientY); moveTo(p.x, p.y); send({ t: 'b', x: p.x, y: p.y, b: [1, 2, 3][e.button] || 1, d: 1 }); }function mouseMove(e) { const p = toScreen(e.clientX, e.clientY); moveTo(p.x, p.y); }function mouseUp(e) { send({ t: 'b', b: [1, 2, 3][e.button] || 1, d: 0 }); releaseMods(); }function wheel(e) { e.preventDefault(); if (e.ctrlKey) { zoomAt(view.z * (e.deltaY < 0 ? 1.15 : 1 / 1.15), e.clientX, e.clientY); return; } wheel.acc = (wheel.acc || 0) + e.deltaY; while (Math.abs(wheel.acc) >= 50) { const s = Math.sign(wheel.acc); send({ t: 'w', dy: s }); wheel.acc -= s * 50; }}// --- keyboard: the phone's (a hidden field) and a real one ---------------------------------------------------------------------const SPECIAL = { Enter: 'Return', Backspace: 'BackSpace', Escape: 'Escape', Tab: 'Tab', Delete: 'Delete', Insert: 'Insert', Home: 'Home', End: 'End', PageUp: 'Prior', PageDown: 'Next', ArrowLeft: 'Left', ArrowRight: 'Right', ArrowUp: 'Up', ArrowDown: 'Down', Control: 'Control_L', Shift: 'Shift_L', Alt: 'Alt_L', AltGraph: 'ISO_Level3_Shift', Meta: 'Super_L', CapsLock: 'Caps_Lock', ContextMenu: 'Menu', PrintScreen: 'Print', ' ': 'space' };for (let i = 1; i <= 12; i++) SPECIAL['F' + i] = 'F' + i;const SENTINEL = ' ';let composing = false;function keyDown(e) { const kbd = e.target === $('kbd'); const special = SPECIAL[e.key]; if (e.key === 'Unidentified' || e.keyCode === 229) return; if (kbd && (e.key === 'Backspace' || (!special && !e.ctrlKey && !e.altKey && !e.metaKey))) return; // the field's input event sends it if (special && kbd && e.key === ' ') return; const k = special || (e.key.length === 1 ? e.key : null); if (!k) return; e.preventDefault(); send({ t: 'k', key: k, d: 1 });}function keyUp(e) { const kbd = e.target === $('kbd'); const special = SPECIAL[e.key]; if (kbd && (e.key === 'Backspace' || e.key === ' ' || (!special && !e.ctrlKey && !e.altKey && !e.metaKey))) return; const k = special || (e.key.length === 1 ? e.key : null); if (k) { send({ t: 'k', key: k, d: 0 }); releaseMods(); }}function kbdInput() { if (composing) return; const el = $('kbd'); const v = el.value; if (v.startsWith(SENTINEL)) { const s = v.slice(SENTINEL.length); if (s) { send({ t: 'txt', s }); releaseMods(); } } else { const lost = SENTINEL.length - Math.min(SENTINEL.length, v.length); for (let i = 0; i < Math.max(1, lost); i++) key('BackSpace'); } el.value = SENTINEL;}// --- stats --------------------------------------------------------------------------------------------------------------------const stats = { rtt: 0, encoder: '', bytes: 0, at: 0, frames: 0 };let decoded = -1; let lastKeyAsk = 0;async function watchdog() { // bytes arrive but nothing is decoded (the first key frame was lost): ask for one if (!pc || input?.readyState !== 'open') return; const r = await pc.getStats(); r.forEach((s) => { if (s.type !== 'inbound-rtp' || s.kind !== 'video') return; const n = s.framesDecoded || 0; if ((n === 0 || n === decoded) && s.bytesReceived > 0 && performance.now() - lastKeyAsk > 1500) { lastKeyAsk = performance.now(); send({ t: 'key' }); } decoded = n; });}async function readStats() { if (!pc || $('stats').hidden) return; const r = await pc.getStats(); let fps = 0; let mbps = 0; let path = ''; let dropped = 0; let res = ''; let delay = 0; r.forEach((s) => { if (s.type === 'inbound-rtp' && s.kind === 'video') { fps = s.framesPerSecond || 0; dropped = s.framesDropped || 0; res = `${s.frameWidth}×${s.frameHeight}`; const now = s.timestamp; if (stats.at) mbps = (s.bytesReceived - stats.bytes) * 8 / ((now - stats.at) * 1000); stats.bytes = s.bytesReceived; stats.at = now; if (s.jitterBufferEmittedCount) delay = Math.round(1000 * s.jitterBufferDelay / s.jitterBufferEmittedCount); } if (s.type === 'candidate-pair' && s.nominated && s.state === 'succeeded') { const local = r.get(s.localCandidateId); const remote = r.get(s.remoteCandidateId); const lan = /^(10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.|127\.|fe80:|fd|fc|::1)/i.test(remote?.address || remote?.ip || ''); path = local?.candidateType === 'relay' || remote?.candidateType === 'relay' ? 'TURN' : lan ? t('path_lan') : t('path_net'); } }); $('stats').textContent = `${res} ${fps} fps ${mbps.toFixed(1)} Mb/s\nRTT ${stats.rtt} ms ${t('buffer')} ${delay} ms\n${path} ${stats.encoder} ${t('dropped')} ${dropped}`;}// --- toolbar -------------------------------------------------------------------------------------------------------------------function setMode(m) { mode = m; store.set('mode', m); $('modeBtn').textContent = t(m); drawCursor();}function wire() { const stage = $('stage'); stage.addEventListener('pointerdown', down); stage.addEventListener('pointermove', move); stage.addEventListener('pointerup', up); stage.addEventListener('pointercancel', up); stage.addEventListener('wheel', wheel, { passive: false }); stage.addEventListener('contextmenu', (e) => e.preventDefault()); for (const id of ['bar', 'keys']) for (const ev of ['pointerdown', 'pointermove', 'pointerup']) $(id).addEventListener(ev, (e) => e.stopPropagation()); addEventListener('keydown', keyDown); addEventListener('keyup', keyUp); const kbd = $('kbd'); kbd.value = SENTINEL; kbd.addEventListener('input', kbdInput); kbd.addEventListener('compositionstart', () => { composing = true; }); kbd.addEventListener('compositionend', (e) => { composing = false; if (e.data) send({ t: 'txt', s: e.data }); kbd.value = SENTINEL; }); addEventListener('resize', () => layout()); $('video').addEventListener('resize', () => layout(true)); $('menuBtn').onclick = () => { $('tools').hidden = !$('tools').hidden; }; $('modeBtn').onclick = () => { setMode(mode === 'touchpad' ? 'direct' : 'touchpad'); toast(t('help_' + mode), 3500); }; $('kbdBtn').onclick = () => { const k = $('kbd'); if (document.activeElement === k) k.blur(); else { k.value = SENTINEL; k.focus(); } }; $('keysBtn').onclick = () => { $('keys').hidden = !$('keys').hidden; document.body.classList.toggle('keys-open', !$('keys').hidden); $('keysBtn').classList.toggle('on', !$('keys').hidden); }; $('fitBtn').onclick = () => layout(true); $('oneBtn').onclick = () => { const p = toStage(cur.x, cur.y); zoomAt(1 / view.f, p.x, p.y); }; $('quality').onchange = () => send({ t: 'kbps', kbps: Number($('quality').value) }); $('statsBtn').onclick = () => { $('stats').hidden = !$('stats').hidden; $('statsBtn').classList.toggle('on', !$('stats').hidden); }; $('fullBtn').onclick = () => { if (document.fullscreenElement) document.exitFullscreen(); else document.documentElement.requestFullscreen?.({ navigationUI: 'hide' }).then(() => screen.orientation?.lock?.('landscape').catch(() => {})).catch(() => {}); }; $('byeBtn').onclick = () => { stop(true); store.set('pairing', null); history.replaceState(null, '', location.pathname + location.search); gate('pair'); }; $('retry').onclick = () => gate('pair'); $('keys').addEventListener('click', (e) => { const b = e.target.closest('button'); if (!b) return; if (b.dataset.k) key(b.dataset.k); else if (b.dataset.b) click(Number(b.dataset.b)); else if (b.dataset.combo) { const ks = b.dataset.combo.split('+'); ks.forEach((k) => send({ t: 'k', key: k, d: 1 })); ks.reverse().forEach((k) => send({ t: 'k', key: k, d: 0 })); releaseMods(); } else if (b.dataset.mod) { const k = b.dataset.mod; const s = mods.get(k); if (!s) { mods.set(k, 'on'); send({ t: 'k', key: k, d: 1 }); b.classList.add('on'); } else if (s === 'on') { mods.set(k, 'lock'); b.classList.add('lock'); } else { mods.delete(k); send({ t: 'k', key: k, d: 0 }); b.classList.remove('on', 'lock'); } } }); setInterval(readStats, 1000); setInterval(watchdog, 1000); setMode(mode);}// --- pairing ---------------------------------------------------------------------------------------------------------------------function pair(text) { const p = Channel.parsePairing(text); if (!p) { $('pairError').textContent = t('bad_link'); $('pairError').hidden = false; return false; } pairing = p; store.set('pairing', text); store.set('device', $('deviceName').value.trim()); start(); return true;}function guessName() { const ua = navigator.userAgent; const m = /Android [\d.]+; ([^;)]+)/.exec(ua); if (m && !/^K$/.test(m[1])) return m[1].trim().slice(0, 40); return /iPhone/.test(ua) ? 'iPhone' : /iPad/.test(ua) ? 'iPad' : /Android/.test(ua) ? 'Android' : 'Browser';}applyTexts();wire();$('deviceName').value = store.get('device') || guessName();$('pairForm').addEventListener('submit', (e) => { e.preventDefault(); $('pairError').hidden = true; pair($('pairLink').value.trim()); });// the link's fragment: this page's, or the framing page's (in the browser the app's page is a frame of /play/, same origin;// the frame may start before that page has its final address: looked for a few seconds)function linkFromAddress() { if (location.hash.includes('r=')) return { href: location.href, w: window }; try { if (window.top !== window && window.top.location.hash.includes('r=')) return { href: window.top.location.href, w: window.top }; } catch { /* another origin */ } return null;}function boot(tries = 0) { const found = linkFromAddress(); const saved = found?.href || store.get('pairing') || ''; if (found || (tries >= 15 && saved)) { if (!Channel.parsePairing(saved)) { gate('pair'); return; } $('pairLink').value = saved; try { if (found) found.w.history.replaceState(null, '', found.w.location.pathname + found.w.location.search); } catch { /* not ours */ } pair(saved); } else if (tries < 15) setTimeout(() => boot(tries + 1), 200); else gate('pair');}gate('wait', t('connecting'));boot();