network: Fetch based browser networking (#1061)

Emulate a networking stack, intercept HTTP requests, serve
them with fetch().  Enable by setting networking_proxy=fetch

Somewhat fixes #198

Just enough networking for http to work for package managers and
net-boot. The networking stack just reflects the macaddress of
packets it receives, so shouldnt be bothered by mac address
changes. Services include:

- ARP replies by establishing the gateways IP
- DHCP server sending gateway, dns, and client-ip
  - Gateway: 192.168.86.1
  - Client: 192.168.86.100
  - Netmask: 255.255.255.0
- UDP DNS server, all addresses resolve to: 192.168.87.1
- NTP Server, giving out the current time from Date()
- Replies to ICMP pings to any IP address
- UDP echo server on port 8

Limitations:
- Request are limited by CORS rules, but CORS-proxies can work around
  this.
- Redirects are handled transparently by fetch. The upside is sites that
  force an http->https upgrade appear to the VM to work over http.

Future Work:
- Allow HTTP requests from browser to be served by VM.
- HTTPS support (maybe with help of a rust based TLS stack in WASM)
- Support for POSTing binary data

* Run devices tests in CI
* Allow multiple Ne2k adapters

Co-authored-by: Fabian <copy@copy.sh>
This commit is contained in:
Rob Blanckaert 2024-07-24 07:22:53 -07:00 committed by GitHub
parent 5435c2f8c4
commit 92c936d632
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 1715 additions and 36 deletions

View file

@ -101,6 +101,9 @@ jobs:
- name: Run integration tests
run: MAX_PARALLEL_TESTS=1 make tests
- name: Run devices tests
run: make devices-test
- name: Run expect tests
run: make expect-tests

View file

@ -87,7 +87,7 @@ CORE_FILES=const.js config.js io.js main.js lib.js buffer.js ide.js pci.js flopp
LIB_FILES=9p.js filesystem.js jor1k.js marshall.js utf8.js
BROWSER_FILES=screen.js keyboard.js mouse.js speaker.js serial.js \
network.js starter.js worker_bus.js dummy_screen.js \
print_stats.js filestorage.js
fetch_network.js print_stats.js filestorage.js
RUST_FILES=$(shell find src/rust/ -name '*.rs') \
src/rust/gen/interpreter.rs src/rust/gen/interpreter0f.rs \
@ -305,6 +305,7 @@ expect-tests: all-debug build/libwabt.js
devices-test: all-debug
./tests/devices/virtio_9p.js
./tests/devices/virtio_console.js
./tests/devices/fetch_network.js
rust-test: $(RUST_FILES)
env RUSTFLAGS="-D warnings" RUST_BACKTRACE=full RUST_TEST_THREADS=1 cargo test -- --nocapture

View file

@ -13,7 +13,7 @@ var CORE_FILES =
"memory.js dma.js pit.js vga.js ps2.js rtc.js uart.js acpi.js apic.js ioapic.js sb16.js " +
"ne2k.js state.js virtio.js virtio_console.js bus.js elf.js kernel.js";
var BROWSER_FILES = "main.js screen.js keyboard.js mouse.js speaker.js serial.js network.js starter.js worker_bus.js print_stats.js filestorage.js";
var BROWSER_FILES = "main.js screen.js keyboard.js mouse.js speaker.js serial.js network.js fetch_network.js starter.js worker_bus.js print_stats.js filestorage.js";
var LIB_FILES = "";
// jor1k stuff

1264
src/browser/fetch_network.js Normal file

File diff suppressed because it is too large Load diff

View file

@ -10,11 +10,13 @@
*
* @param {string} url
* @param {BusConnector} bus
* @param {number} [id=0] id
*/
function NetworkAdapter(url, bus)
function NetworkAdapter(url, bus, id)
{
this.bus = bus;
this.socket = undefined;
this.id = id || 0;
// TODO: circular buffer?
this.send_queue = [];
@ -24,7 +26,7 @@ function NetworkAdapter(url, bus)
this.last_connect_attempt = Date.now() - this.reconnect_interval;
this.send_queue_limit = 64;
this.bus.register("net0-send", function(data)
this.bus.register("net" + this.id + "-send", function(data)
{
this.send(data);
}, this);
@ -34,7 +36,7 @@ NetworkAdapter.prototype.handle_message = function(e)
{
if(this.bus)
{
this.bus.send("net0-receive", new Uint8Array(e.data));
this.bus.send("net" + this.id + "-receive", new Uint8Array(e.data));
}
};

View file

@ -293,7 +293,14 @@ V86.prototype.continue_init = async function(emulator, options)
}
else if(options.network_relay_url)
{
this.network_adapter = new NetworkAdapter(options.network_relay_url, this.bus);
if(options.network_relay_url === "fetch")
{
this.network_adapter = new FetchNetworkAdapter(this.bus);
}
else
{
this.network_adapter = new NetworkAdapter(options.network_relay_url, this.bus);
}
}
// Enable unconditionally, so that state images don't miss hardware

View file

@ -4,30 +4,31 @@ var
/** @const */ LOG_ALL = -1,
/** @const */ LOG_NONE = 0,
/** @const */ LOG_OTHER = 0x000001,
/** @const */ LOG_CPU = 0x000002,
/** @const */ LOG_FPU = 0x000004,
/** @const */ LOG_MEM = 0x000008,
/** @const */ LOG_DMA = 0x000010,
/** @const */ LOG_IO = 0x000020,
/** @const */ LOG_PS2 = 0x000040,
/** @const */ LOG_PIC = 0x000080,
/** @const */ LOG_VGA = 0x000100,
/** @const */ LOG_PIT = 0x000200,
/** @const */ LOG_MOUSE = 0x000400,
/** @const */ LOG_PCI = 0x000800,
/** @const */ LOG_BIOS = 0x001000,
/** @const */ LOG_FLOPPY = 0x002000,
/** @const */ LOG_SERIAL = 0x004000,
/** @const */ LOG_DISK = 0x008000,
/** @const */ LOG_RTC = 0x010000,
// unused 0x020000,
/** @const */ LOG_ACPI = 0x040000,
/** @const */ LOG_APIC = 0x080000,
/** @const */ LOG_NET = 0x100000,
/** @const */ LOG_VIRTIO = 0x200000,
/** @const */ LOG_9P = 0x400000,
/** @const */ LOG_SB16 = 0x800000;
/** @const */ LOG_OTHER = 0x0000001,
/** @const */ LOG_CPU = 0x0000002,
/** @const */ LOG_FPU = 0x0000004,
/** @const */ LOG_MEM = 0x0000008,
/** @const */ LOG_DMA = 0x0000010,
/** @const */ LOG_IO = 0x0000020,
/** @const */ LOG_PS2 = 0x0000040,
/** @const */ LOG_PIC = 0x0000080,
/** @const */ LOG_VGA = 0x0000100,
/** @const */ LOG_PIT = 0x0000200,
/** @const */ LOG_MOUSE = 0x0000400,
/** @const */ LOG_PCI = 0x0000800,
/** @const */ LOG_BIOS = 0x0001000,
/** @const */ LOG_FLOPPY = 0x0002000,
/** @const */ LOG_SERIAL = 0x0004000,
/** @const */ LOG_DISK = 0x0008000,
/** @const */ LOG_RTC = 0x0010000,
// unused 0x0020000,
/** @const */ LOG_ACPI = 0x0040000,
/** @const */ LOG_APIC = 0x0080000,
/** @const */ LOG_NET = 0x0100000,
/** @const */ LOG_VIRTIO = 0x0200000,
/** @const */ LOG_9P = 0x0400000,
/** @const */ LOG_SB16 = 0x0800000,
/** @const */ LOG_FETCH = 0x1000000;
/**
@ -57,7 +58,8 @@ var LOG_NAMES = [
[LOG_NET, "NET"],
[LOG_VIRTIO, "VIO"],
[LOG_9P, "9P"],
[LOG_SB16, "SB16"]
[LOG_SB16, "SB16"],
[LOG_FETCH, "FETC"],
];
var

View file

@ -286,8 +286,9 @@ function dump_packet(packet, prefix)
* @param {BusConnector} bus
* @param {Boolean} preserve_mac_from_state_image
* @param {Boolean} mac_address_translation
* @param {number} [id=0] id
*/
function Ne2k(cpu, bus, preserve_mac_from_state_image, mac_address_translation)
function Ne2k(cpu, bus, preserve_mac_from_state_image, mac_address_translation, id)
{
/** @const @type {CPU} */
this.cpu = cpu;
@ -295,17 +296,18 @@ function Ne2k(cpu, bus, preserve_mac_from_state_image, mac_address_translation)
/** @const @type {PCI} */
this.pci = cpu.devices.pci;
this.id = id || 0;
this.preserve_mac_from_state_image = preserve_mac_from_state_image;
this.mac_address_translation = mac_address_translation;
/** @const @type {BusConnector} */
this.bus = bus;
this.bus.register("net0-receive", function(data)
this.bus.register("net" + this.id + "-receive", function(data)
{
this.receive(data);
}, this);
this.port = 0x300;
this.port = 0x300 + 0x100 * this.id;
this.name = "ne2k";
@ -320,7 +322,7 @@ function Ne2k(cpu, bus, preserve_mac_from_state_image, mac_address_translation)
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf4, 0x1a, 0x00, 0x11,
0x00, 0x00, 0xb8, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00,
];
this.pci_id = 0x05 << 3;
this.pci_id = (this.id === 0 ? 0x05 : (0x07 + this.id)) << 3;
this.pci_bars = [
{
size: 32,
@ -353,6 +355,8 @@ function Ne2k(cpu, bus, preserve_mac_from_state_image, mac_address_translation)
Math.random() * 255 | 0,
]);
this.bus.send("net" + this.id + "-mac", format_mac(this.mac));
// multicast addresses
this.mar = Uint8Array.of(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF);
@ -419,7 +423,7 @@ function Ne2k(cpu, bus, preserve_mac_from_state_image, mac_address_translation)
translate_mac_address(data, this.mac_address_in_state, this.mac);
}
this.bus.send("net0-send", data);
this.bus.send("net" + this.id + "-send", data);
this.bus.send("eth-transmit-end", [data.length]);
this.cr &= ~4;
this.do_interrupt(ENISR_TX);
@ -1106,6 +1110,7 @@ Ne2k.prototype.set_state = function(state)
" guest_os_mac=" + format_mac(this.mac_address_in_state) +
" real_mac=" + format_mac(this.mac), LOG_NET);
}
this.bus.send("net" + this.id + "-mac", format_mac(this.mac));
};
Ne2k.prototype.do_interrupt = function(ir_mask)

352
tests/devices/fetch_network.js Executable file
View file

@ -0,0 +1,352 @@
#!/usr/bin/env node
"use strict";
process.on("unhandledRejection", exn => { throw exn; });
const TEST_RELEASE_BUILD = +process.env.TEST_RELEASE_BUILD;
const V86 = require(`../../build/${TEST_RELEASE_BUILD ? "libv86" : "libv86-debug"}.js`).V86;
const assert = require("assert").strict;
const SHOW_LOGS = false;
const STOP_ON_FIRST_FAILURE = false;
function log_pass(msg, ...args)
{
console.log(`\x1b[92m[+] ${msg}\x1b[0m`, ...args);
}
function log_warn(msg, ...args)
{
console.error(`\x1b[93m[!] ${msg}\x1b[0m`, ...args);
}
function log_fail(msg, ...args)
{
console.error(`\x1b[91m[-] ${msg}\x1b[0m`, ...args);
}
const tests =
[
{
name: "DHCP",
timeout: 60,
start: () =>
{
emulator.serial0_send("udhcpc\n");
emulator.serial0_send("echo -e done\\\\tudhcpc\n");
},
end_trigger: "done\tudhcpc",
end: (capture) =>
{
assert(/lease of 192.168.86.100 obtained/.test(capture), "lease of 192.168.86.100 obtained");
},
},
{
name: "ifconfig",
timeout: 60,
start: () =>
{
emulator.serial0_send("ifconfig\n");
emulator.serial0_send("echo -e done\\\\tifconfig\n");
},
end_trigger: "done\tifconfig",
end: (capture) =>
{
assert(/192.168.86.100/.test(capture), "192.168.86.100");
},
},
{
name: "route",
timeout: 60,
start: () =>
{
emulator.serial0_send("ip route\n");
emulator.serial0_send("echo -e done\\\\troute\n");
},
end_trigger: "done\troute",
end: (capture) =>
{
assert(/192.168.86.1/.test(capture), "192.168.86.100");
},
},
{
name: "ping 1.2.3.4",
timeout: 60,
start: () =>
{
emulator.serial0_send("ping -c 2 1.2.3.4\n");
emulator.serial0_send("echo -e done\\\\tping\n");
},
end_trigger: "done\tping",
end: (capture) =>
{
assert(/2 packets transmitted, 2 packets received, 0% packet loss/.test(capture), "2 packets transmitted, 2 packets received, 0% packet loss");
},
},
{
name: "arp -a",
timeout: 60,
start: () =>
{
emulator.serial0_send("arp -a\n");
emulator.serial0_send("echo -e done\\\\tarp\n");
},
end_trigger: "done\tarp",
end: (capture) =>
{
assert(/.192.168.86.1. at 52:54:00:01:02:03 \[ether\] {2}on eth0/.test(capture), "(192.168.86.1) at 52:54:00:01:02:03 [ether] on eth0");
},
},
{
name: "Curl mocked.example.org",
timeout: 60,
allow_failure: true,
start: () =>
{
emulator.serial0_send("wget -T 10 -O - mocked.example.org\n");
emulator.serial0_send("echo -e done\\\\tmocked.example.org\n");
},
end_trigger: "done\tmocked.example.org",
end: (capture) =>
{
assert(/This text is from the mock/.test(capture), "got mocked.example.org text");
},
},
{
name: "Curl example.org",
timeout: 60,
allow_failure: true,
start: () =>
{
emulator.serial0_send("wget -T 10 -O - example.org\n");
emulator.serial0_send("echo -e done\\\\texample.org\n");
},
end_trigger: "done\texample.org",
end: (capture) =>
{
assert(/This domain is for use in illustrative examples in documents/.test(capture), "got example.org text");
},
},
];
let test_num = 0;
let test_timeout = 0;
const failed_tests = [];
const emulator = new V86({
bios: { url: __dirname + "/../../bios/seabios.bin" },
vga_bios: { url: __dirname + "/../../bios/vgabios.bin" },
cdrom: { url: __dirname + "/../../images/linux4.iso" },
autostart: true,
memory_size: 64 * 1024 * 1024,
disable_jit: +process.env.DISABLE_JIT,
network_relay_url: "fetch",
log_level: SHOW_LOGS ? 0x400000 : 0,
});
emulator.add_listener("emulator-ready", function () {
let network_adapter = emulator.network_adapter;
let original_fetch = network_adapter.fetch;
network_adapter.fetch = (url, opts) => {
if(/^http:\/\/mocked.example.org\/?/.test(url)) {
let contents = new TextEncoder().encode("This text is from the mock");
let headers = new Headers();
return new Promise(res => setTimeout(() => res([
{status: 200, statusText: "OK", headers: headers},
contents.buffer
]), 50));
}
return original_fetch.call(network_adapter, url, opts);
};
});
let ran_command = false;
let line = "";
let capturing = false;
let capture = "";
let next_trigger;
let next_trigger_handler;
function start_timeout()
{
if(tests[test_num].timeout)
{
test_timeout = setTimeout(() =>
{
log_fail("Test #%d (%s) took longer than %s sec. Timing out and terminating.", test_num, tests[test_num].name, tests[test_num].timeout);
process.exit(1);
}, tests[test_num].timeout * 1000);
}
}
function begin()
{
start_timeout();
console.log("\nPreparing test #%d: %s", test_num, tests[test_num].name);
start_test();
}
function start_test()
{
console.log("Starting test #%d: %s", test_num, tests[test_num].name);
capture = "";
tests[test_num].start();
if(tests[test_num].capture_trigger)
{
next_trigger = tests[test_num].capture_trigger;
next_trigger_handler = start_capture;
}
else
{
next_trigger = tests[test_num].end_trigger;
next_trigger_handler = end_test;
}
start_capture();
}
function start_capture()
{
console.log("Capturing...");
capture = "";
capturing = true;
next_trigger = tests[test_num].end_trigger;
next_trigger_handler = end_test;
}
function end_test()
{
capturing = false;
if(tests[test_num].timeout)
{
clearTimeout(test_timeout);
}
let test_has_failed = false;
try {
tests[test_num].end(capture);
} catch(e) {
console.log(e);
test_has_failed = true;
}
if(!test_has_failed)
{
log_pass("Test #%d passed: %s", test_num, tests[test_num].name);
}
else
{
if(tests[test_num].allow_failure)
{
log_warn("Test #%d failed: %s (failure allowed)", test_num, tests[test_num].name);
}
else
{
log_fail("Test #%d failed: %s", test_num, tests[test_num].name);
if(STOP_ON_FIRST_FAILURE)
{
finish_tests();
}
}
test_has_failed = false;
}
test_num++;
if(test_num < tests.length)
{
begin();
}
else
{
finish_tests();
}
}
function finish_tests()
{
emulator.stop();
console.log("\nTests finished.");
if(failed_tests.length === 0)
{
console.log("All tests passed");
}
else
{
let unallowed_failure = false;
console.error("Failed %d out of %d tests:", failed_tests.length, tests.length);
for(const num of failed_tests)
{
if(tests[num].allow_failure)
{
log_warn("#%d %s (failure allowed)", num, tests[num].name);
}
else
{
unallowed_failure = true;
log_fail("#%d %s", num, tests[num].name);
}
}
if(unallowed_failure)
{
process.exit(1);
}
}
}
emulator.bus.register("emulator-started", function()
{
console.error("Booting now, please stand by");
});
emulator.add_listener("serial0-output-byte", function(byte)
{
const chr = String.fromCharCode(byte);
if(chr < " " && chr !== "\n" && chr !== "\t" || chr > "~")
{
return;
}
let new_line = "";
let is_new_line = false;
if(chr === "\n")
{
is_new_line = true;
new_line = line;
line = "";
}
else
{
line += chr;
}
if(!ran_command && line.endsWith("~% "))
{
ran_command = true;
begin();
}
else if(new_line === next_trigger)
{
next_trigger_handler();
}
else if(is_new_line && capturing)
{
capture += new_line + "\n";
console.log(" Captured: %s", new_line);
}
else if(is_new_line)
{
console.log(" Serial: %s", new_line);
}
});

View file

@ -0,0 +1,43 @@
const dgram = require("node:dgram");
const server = dgram.createSocket("udp4");
const fs = require("node:fs");
const path = require("node:path");
// qemu-system-i386 -m 2G -nographic -hda ~/disk.qcow2 -netdev dgram,id=net0,remote.type=inet,remote.host=127.0.0.1,remote.port=6677,local.host=127.0.0.1,local.port=7744,local.type=inet -device e1000,netdev=net0
globalThis.dbg_assert = require("node:assert");
globalThis.dbg_log = (what, level) => console.log(what);
globalThis.LOG_NET = 0;
const { FetchNetworkAdapter } = require(path.join(__dirname, "..", "src", "browser", "fetch_network.js"));
const events = {};
const bus = {
register: (name, fn, bind) => events[name] = fn.bind(bind),
send: (name, data) => events[name](data)
};
const a = new FetchNetworkAdapter(bus);
server.on("error", (err) => {
console.error(`server error:\n${err.stack}`);
server.close();
});
server.on("message", (msg, rinfo) => {
//console.log(`server got: ${msg.toString("hex")} from ${rinfo.address}:${rinfo.port}`);
events["net0-receive"] = (data) => {
//console.log(`sending: ${Buffer.from(data).toString("hex")} to ${rinfo.address}:${rinfo.port}`);
server.send(Buffer.from(data), rinfo.port);
};
bus.send("net0-send", new Uint8Array(msg));
});
server.on("listening", () => {
const address = server.address();
console.log(`server listening ${address.address}:${address.port}`);
});
server.bind(6677);