From f5bf3559650c0b0fbe48375baef71ad3270e0cb7 Mon Sep 17 00:00:00 2001 From: Radu Matei Date: Tue, 2 Nov 2021 01:40:52 -0700 Subject: [PATCH] Add a very simple Engine This commit adds a very simple engine implementation that can be used to generate an execution context based on a WITX definition, and run WebAssembly components that implement the WITX. The implementation uses Wasmtime and `witx-bindgen`, and the test is an echo implementation written in Rust. Currently, the implementation does not test additional features, such as custom linker imports. Signed-off-by: Radu Matei --- .gitignore | 2 +- Cargo.lock | 72 ++++++++++- build.rs | 68 ++++++++++ crates/engine/Cargo.toml | 4 + crates/engine/src/lib.rs | 156 +++++++++++++++++++++-- crates/engine/tests/echo.witx | 1 + crates/engine/tests/rust-echo/Cargo.lock | 140 ++++++++++++++++++++ crates/engine/tests/rust-echo/Cargo.toml | 13 ++ crates/engine/tests/rust-echo/src/lib.rs | 9 ++ 9 files changed, 450 insertions(+), 15 deletions(-) create mode 100644 build.rs create mode 100644 crates/engine/tests/echo.witx create mode 100644 crates/engine/tests/rust-echo/Cargo.lock create mode 100644 crates/engine/tests/rust-echo/Cargo.toml create mode 100644 crates/engine/tests/rust-echo/src/lib.rs diff --git a/.gitignore b/.gitignore index ea8c4bf7..eb5a316c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -/target +target diff --git a/Cargo.lock b/Cargo.lock index 52e0450e..83d819bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -594,7 +594,9 @@ dependencies = [ "wasi-common", "wasi-experimental-http-wasmtime", "wasmtime", + "wasmtime-wasi", "witx-bindgen-rust", + "witx-bindgen-wasmtime", ] [[package]] @@ -2716,7 +2718,16 @@ version = "0.1.0" source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0#0fea5183d121fdf736585ab9d291b3cc08487cd0" dependencies = [ "anyhow", - "witx2", + "witx2 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0)", +] + +[[package]] +name = "witx-bindgen-gen-core" +version = "0.1.0" +source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" +dependencies = [ + "anyhow", + "witx2 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e)", ] [[package]] @@ -2725,7 +2736,16 @@ version = "0.1.0" source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0#0fea5183d121fdf736585ab9d291b3cc08487cd0" dependencies = [ "heck", - "witx-bindgen-gen-core", + "witx-bindgen-gen-core 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0)", +] + +[[package]] +name = "witx-bindgen-gen-rust" +version = "0.1.0" +source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" +dependencies = [ + "heck", + "witx-bindgen-gen-core 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e)", ] [[package]] @@ -2734,8 +2754,18 @@ version = "0.1.0" source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0#0fea5183d121fdf736585ab9d291b3cc08487cd0" dependencies = [ "heck", - "witx-bindgen-gen-core", - "witx-bindgen-gen-rust", + "witx-bindgen-gen-core 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0)", + "witx-bindgen-gen-rust 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0)", +] + +[[package]] +name = "witx-bindgen-gen-wasmtime" +version = "0.1.0" +source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" +dependencies = [ + "heck", + "witx-bindgen-gen-core 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e)", + "witx-bindgen-gen-rust 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e)", ] [[package]] @@ -2754,10 +2784,33 @@ source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121f dependencies = [ "proc-macro2", "syn", - "witx-bindgen-gen-core", + "witx-bindgen-gen-core 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0)", "witx-bindgen-gen-rust-wasm", ] +[[package]] +name = "witx-bindgen-wasmtime" +version = "0.1.0" +source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" +dependencies = [ + "anyhow", + "bitflags", + "thiserror", + "wasmtime", + "witx-bindgen-wasmtime-impl", +] + +[[package]] +name = "witx-bindgen-wasmtime-impl" +version = "0.1.0" +source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" +dependencies = [ + "proc-macro2", + "syn", + "witx-bindgen-gen-core 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e)", + "witx-bindgen-gen-wasmtime", +] + [[package]] name = "witx2" version = "0.1.0" @@ -2767,6 +2820,15 @@ dependencies = [ "id-arena", ] +[[package]] +name = "witx2" +version = "0.1.0" +source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" +dependencies = [ + "anyhow", + "id-arena", +] + [[package]] name = "zstd" version = "0.9.0+zstd.1.5.0" diff --git a/build.rs b/build.rs new file mode 100644 index 00000000..03ef50fa --- /dev/null +++ b/build.rs @@ -0,0 +1,68 @@ +use std::{ + collections::HashMap, + process::{self, Command}, +}; + +const ECHO_WITX: &str = "crates/engine/tests/echo.witx"; +const ECHO_RUST: &str = "crates/engine/tests/rust-echo"; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed={}", ECHO_WITX); + println!("cargo:rerun-if-changed={}/src/lib.rs", ECHO_RUST); + + cargo_build(ECHO_RUST); +} + +fn cargo_build(dir: &str) { + run( + vec!["cargo", "build", "--target", "wasm32-wasi", "--release"], + Some(dir), + None, + ); +} + +fn run + AsRef>( + args: Vec, + dir: Option, + env: Option>, +) { + let mut cmd = Command::new(get_os_process()); + cmd.stdout(process::Stdio::piped()); + cmd.stderr(process::Stdio::piped()); + + if let Some(dir) = dir { + cmd.current_dir(dir.into()); + }; + + if let Some(env) = env { + for (k, v) in env { + cmd.env(k, v); + } + }; + + cmd.arg("-c"); + cmd.arg( + args.into_iter() + .map(Into::into) + .collect::>() + .join(" "), + ); + + let output = cmd.output().unwrap(); + let code = output.status.code().unwrap(); + if code != 0 { + println!("{:#?}", std::str::from_utf8(&output.stderr).unwrap()); + println!("{:#?}", std::str::from_utf8(&output.stdout).unwrap()); + // just fail + assert_eq!(0, code); + } +} + +fn get_os_process() -> String { + if cfg!(target_os = "windows") { + String::from("powershell.exe") + } else { + String::from("/bin/bash") + } +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index c8368f89..4d86a191 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -22,4 +22,8 @@ wasi-common = "0.30" wasi-experimental-http-wasmtime = "0.6.0" wasmtime = "0.30" + wasmtime-wasi = "0.30" witx-bindgen-rust = { git = "https://github.com/bytecodealliance/witx-bindgen", rev = "0fea5183d121fdf736585ab9d291b3cc08487cd0" } + +[dev-dependencies] + witx-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/witx-bindgen", rev = "b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 2fdb5576..71a3d66a 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -2,9 +2,12 @@ #![deny(missing_docs)] +use anyhow::Result; use std::sync::Arc; use wasi_common::WasiCtx; -use wasmtime::{Engine, InstancePre}; +use wasi_experimental_http_wasmtime::HttpCtx; +use wasmtime::{Engine, Instance, InstancePre, Linker, Module, Store}; +use wasmtime_wasi::sync::{ambient_authority, Dir, WasiCtxBuilder}; /// Engine configuration. #[derive(Clone, Default)] @@ -26,15 +29,26 @@ impl Config { preopen_dirs: Vec<(String, String)>, allowed_http_hosts: Option>, ) -> Self { + Self { + env_vars, + preopen_dirs, + allowed_http_hosts, + ..Default::default() + } + } + + /// Create a default instance of the configuration instance. + pub fn default() -> Self { + // In order for Wasmtime to run WebAssembly components, multi memory + // and module linking must always be enabled. + // See https://github.com/bytecodealliance/witx-bindgen/blob/main/crates/wasmlink let mut wasmtime_config = wasmtime::Config::default(); wasmtime_config.wasm_multi_memory(true); wasmtime_config.wasm_module_linking(true); Self { - env_vars, - preopen_dirs, - allowed_http_hosts, wasmtime_config, + ..Default::default() } } } @@ -48,15 +62,139 @@ pub struct RuntimeContext { pub data: Option, } -/// The generic execution context. -#[derive(Clone)] -pub struct ExecutionContext { +/// Builder for the execution context. +#[derive(Default)] +pub struct ExecutionContextBuilder { /// Entrypoint of the WebAssembly compnent. pub entrypoint_path: String, /// Top-level runtime configuration. pub config: Config, - /// Pre-initialized WebAssembly instance. - pub pre: Arc>>, + /// Linker used to configure the execution context. + pub linker: Linker>, + /// Store used to configure the execution context. + pub store: Store>, /// Wasmtime engine. pub engine: Engine, } + +impl ExecutionContextBuilder { + /// Create a new instance of the execution builder. + pub fn new(entrypoint_path: String, config: Config) -> Result> { + let data = RuntimeContext::default(); + let engine = Engine::new(&config.wasmtime_config)?; + let store = Store::new(&engine, data); + let linker = Linker::new(&engine); + + Ok(Self { + entrypoint_path, + config, + linker, + store, + engine, + }) + } + + /// Configure the WASI linker imports for the current execution context. + pub fn link_wasi<'a>(&'a mut self) -> Result<&'a Self> { + wasmtime_wasi::add_to_linker(&mut self.linker, |ctx| ctx.wasi.as_mut().unwrap())?; + Ok(self) + } + + /// Configure the HTTP linker imports for the current execution context. + pub fn link_http<'a>(&'a mut self) -> Result<&'a Self> { + let hosts = &self.config.allowed_http_hosts.clone(); + let http = HttpCtx::new(hosts.clone(), None)?; + http.add_to_linker(&mut self.linker)?; + Ok(self) + } + + /// Build a new instance of the execution context by pre-instantiating the entrypoint module. + pub fn build(self) -> Result> { + let module = Module::from_file(&self.engine, &self.entrypoint_path)?; + let pre = self.linker.instantiate_pre(self.store, &module)?; + Ok(ExecutionContext { + config: self.config, + engine: self.engine, + pre: Arc::new(pre), + }) + } + + /// Build a new default instance of the execution context by pre-instantiating the entrypoint module. + pub fn build_default(entrypoint_path: &str, config: Config) -> Result> { + let mut builder = Self::new(entrypoint_path.into(), config)?; + builder.link_wasi()?; + builder.link_http()?; + + builder.build() + } +} + +/// The generic execution context. +#[derive(Clone)] +pub struct ExecutionContext { + /// Top-level runtime configuration. + pub config: Config, + /// Wasmtime engine. + pub engine: Engine, + /// Pre-initialized WebAssembly instance. + pub pre: Arc>>, +} + +impl ExecutionContext { + /// Prepare an instance with actual data + pub fn prepare(&self, data: Option) -> Result<(Store>, Instance)> { + let mut store = self.make_store(data)?; + let instance = self.pre.instantiate(&mut store)?; + Ok((store, instance)) + } + + fn make_store(&self, data: Option) -> Result>> { + let mut ctx = RuntimeContext::default(); + ctx.data = data; + let mut wasi_ctx = WasiCtxBuilder::new() + .inherit_stdio() + .envs(&self.config.env_vars)?; + + for (guest, host) in &self.config.preopen_dirs { + wasi_ctx = + wasi_ctx.preopened_dir(Dir::open_ambient_dir(host, ambient_authority())?, guest)?; + } + + ctx.wasi = Some(wasi_ctx.build()); + + let store = Store::new(&self.engine, ctx); + Ok(store) + } +} + +#[cfg(test)] +mod test { + use super::*; + use anyhow::Result; + use echo::*; + use std::sync::Arc; + + witx_bindgen_wasmtime::import!("crates/engine/tests/echo.witx"); + const RUST_ENTRYPOINT_PATH: &str = "tests/rust-echo/target/wasm32-wasi/release/echo.wasm"; + + #[derive(Clone)] + pub struct EchoEngine(pub Arc>); + + impl EchoEngine { + pub fn execute(&self, msg: &str) -> Result { + let (mut store, instance) = self.0.prepare(None)?; + let e = Echo::new(&mut store, &instance, |host| host.data.as_mut().unwrap())?; + let res = e.echo(&mut store, msg)?; + Ok(res) + } + } + + #[test] + fn test_rust_echo() { + let e = ExecutionContextBuilder::build_default(RUST_ENTRYPOINT_PATH, Config::default()) + .unwrap(); + let e = EchoEngine(Arc::new(e)); + + assert_eq!(e.execute("Fermyon").unwrap(), "Hello, Fermyon".to_string()); + } +} diff --git a/crates/engine/tests/echo.witx b/crates/engine/tests/echo.witx new file mode 100644 index 00000000..f237bd34 --- /dev/null +++ b/crates/engine/tests/echo.witx @@ -0,0 +1 @@ +echo: function(msg: string) -> string diff --git a/crates/engine/tests/rust-echo/Cargo.lock b/crates/engine/tests/rust-echo/Cargo.lock new file mode 100644 index 00000000..9b7b29c1 --- /dev/null +++ b/crates/engine/tests/rust-echo/Cargo.lock @@ -0,0 +1,140 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1" + +[[package]] +name = "async-trait" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "echo" +version = "0.1.0" +dependencies = [ + "witx-bindgen-rust", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + +[[package]] +name = "proc-macro2" +version = "1.0.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba508cc11742c0dc5c1659771673afbab7a0efab23aa17e854cbab0837ed0b43" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2afee18b8beb5a596ecb4a2dce128c719b4ba399d34126b9e4396e3f9860966" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "unicode-segmentation" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "witx-bindgen-gen-core" +version = "0.1.0" +source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" +dependencies = [ + "anyhow", + "witx2", +] + +[[package]] +name = "witx-bindgen-gen-rust" +version = "0.1.0" +source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" +dependencies = [ + "heck", + "witx-bindgen-gen-core", +] + +[[package]] +name = "witx-bindgen-gen-rust-wasm" +version = "0.1.0" +source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" +dependencies = [ + "heck", + "witx-bindgen-gen-core", + "witx-bindgen-gen-rust", +] + +[[package]] +name = "witx-bindgen-rust" +version = "0.1.0" +source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" +dependencies = [ + "async-trait", + "witx-bindgen-rust-impl", +] + +[[package]] +name = "witx-bindgen-rust-impl" +version = "0.1.0" +source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" +dependencies = [ + "proc-macro2", + "syn", + "witx-bindgen-gen-core", + "witx-bindgen-gen-rust-wasm", +] + +[[package]] +name = "witx2" +version = "0.1.0" +source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" +dependencies = [ + "anyhow", + "id-arena", +] diff --git a/crates/engine/tests/rust-echo/Cargo.toml b/crates/engine/tests/rust-echo/Cargo.toml new file mode 100644 index 00000000..bc67d84d --- /dev/null +++ b/crates/engine/tests/rust-echo/Cargo.toml @@ -0,0 +1,13 @@ +[package] + name = "echo" + version = "0.1.0" + edition = "2021" + authors = ["Radu Matei "] + +[lib] + crate-type = ["cdylib"] + +[dependencies] + witx-bindgen-rust = { git = "https://github.com/bytecodealliance/witx-bindgen", rev = "b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" } + +[workspace] diff --git a/crates/engine/tests/rust-echo/src/lib.rs b/crates/engine/tests/rust-echo/src/lib.rs new file mode 100644 index 00000000..816cbebb --- /dev/null +++ b/crates/engine/tests/rust-echo/src/lib.rs @@ -0,0 +1,9 @@ +witx_bindgen_rust::export!("../echo.witx"); + +struct Echo {} + +impl echo::Echo for Echo { + fn echo(msg: String) -> String { + format!("Hello, {}", msg) + } +}