Add a very simple Engine

This commit adds a very simple engine implementation that can be used to generate an execution context based on a WITX definition, and run WebAssembly components that implement the WITX. The implementation uses Wasmtime and `witx-bindgen`, and the test is an echo implementation written in Rust. Currently, the implementation does not test additional features, such as custom linker imports. Signed-off-by: Radu Matei <radu.matei@fermyon.com>
2021-11-02 01:40:52 -07:00 · 2021-11-02 01:40:52 -07:00 · f5bf355965
parent 2767973bd7
commit f5bf355965
9 changed files with 450 additions and 15 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1 +1 @@
-/target
+target
--- a/Cargo.lock
+++ b/Cargo.lock
@ -594,7 +594,9 @@ dependencies = [
 "wasi-common",
 "wasi-experimental-http-wasmtime",
 "wasmtime",
+ "wasmtime-wasi",
 "witx-bindgen-rust",
+ "witx-bindgen-wasmtime",
 ]

 [[package]]
@ -2716,7 +2718,16 @@ version = "0.1.0"
 source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0#0fea5183d121fdf736585ab9d291b3cc08487cd0"
 dependencies = [
 "anyhow",
- "witx2",
+ "witx2 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0)",
+]
+
+[[package]]
+name = "witx-bindgen-gen-core"
+version = "0.1.0"
+source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e"
+dependencies = [
+ "anyhow",
+ "witx2 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e)",
 ]

 [[package]]
@ -2725,7 +2736,16 @@ version = "0.1.0"
 source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0#0fea5183d121fdf736585ab9d291b3cc08487cd0"
 dependencies = [
 "heck",
- "witx-bindgen-gen-core",
+ "witx-bindgen-gen-core 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0)",
+]
+
+[[package]]
+name = "witx-bindgen-gen-rust"
+version = "0.1.0"
+source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e"
+dependencies = [
+ "heck",
+ "witx-bindgen-gen-core 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e)",
 ]

 [[package]]
@ -2734,8 +2754,18 @@ version = "0.1.0"
 source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0#0fea5183d121fdf736585ab9d291b3cc08487cd0"
 dependencies = [
 "heck",
- "witx-bindgen-gen-core",
- "witx-bindgen-gen-rust",
+ "witx-bindgen-gen-core 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0)",
+ "witx-bindgen-gen-rust 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0)",
+]
+
+[[package]]
+name = "witx-bindgen-gen-wasmtime"
+version = "0.1.0"
+source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e"
+dependencies = [
+ "heck",
+ "witx-bindgen-gen-core 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e)",
+ "witx-bindgen-gen-rust 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e)",
 ]

 [[package]]
@ -2754,10 +2784,33 @@ source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121f
 dependencies = [
 "proc-macro2",
 "syn",
- "witx-bindgen-gen-core",
+ "witx-bindgen-gen-core 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=0fea5183d121fdf736585ab9d291b3cc08487cd0)",
 "witx-bindgen-gen-rust-wasm",
 ]

+[[package]]
+name = "witx-bindgen-wasmtime"
+version = "0.1.0"
+source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e"
+dependencies = [
+ "anyhow",
+ "bitflags",
+ "thiserror",
+ "wasmtime",
+ "witx-bindgen-wasmtime-impl",
+]
+
+[[package]]
+name = "witx-bindgen-wasmtime-impl"
+version = "0.1.0"
+source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e"
+dependencies = [
+ "proc-macro2",
+ "syn",
+ "witx-bindgen-gen-core 0.1.0 (git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e)",
+ "witx-bindgen-gen-wasmtime",
+]
+
 [[package]]
 name = "witx2"
 version = "0.1.0"
@ -2767,6 +2820,15 @@ dependencies = [
 "id-arena",
 ]

+[[package]]
+name = "witx2"
+version = "0.1.0"
+source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e"
+dependencies = [
+ "anyhow",
+ "id-arena",
+]
+
 [[package]]
 name = "zstd"
 version = "0.9.0+zstd.1.5.0"
--- a/build.rs
+++ b/build.rs
@ -0,0 +1,68 @@
+use std::{
+    collections::HashMap,
+    process::{self, Command},
+};
+
+const ECHO_WITX: &str = "crates/engine/tests/echo.witx";
+const ECHO_RUST: &str = "crates/engine/tests/rust-echo";
+
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-changed={}", ECHO_WITX);
+    println!("cargo:rerun-if-changed={}/src/lib.rs", ECHO_RUST);
+
+    cargo_build(ECHO_RUST);
+}
+
+fn cargo_build(dir: &str) {
+    run(
+        vec!["cargo", "build", "--target", "wasm32-wasi", "--release"],
+        Some(dir),
+        None,
+    );
+}
+
+fn run<S: Into<String> + AsRef<std::ffi::OsStr>>(
+    args: Vec<S>,
+    dir: Option<S>,
+    env: Option<HashMap<S, S>>,
+) {
+    let mut cmd = Command::new(get_os_process());
+    cmd.stdout(process::Stdio::piped());
+    cmd.stderr(process::Stdio::piped());
+
+    if let Some(dir) = dir {
+        cmd.current_dir(dir.into());
+    };
+
+    if let Some(env) = env {
+        for (k, v) in env {
+            cmd.env(k, v);
+        }
+    };
+
+    cmd.arg("-c");
+    cmd.arg(
+        args.into_iter()
+            .map(Into::into)
+            .collect::<Vec<String>>()
+            .join(" "),
+    );
+
+    let output = cmd.output().unwrap();
+    let code = output.status.code().unwrap();
+    if code != 0 {
+        println!("{:#?}", std::str::from_utf8(&output.stderr).unwrap());
+        println!("{:#?}", std::str::from_utf8(&output.stdout).unwrap());
+        // just fail
+        assert_eq!(0, code);
+    }
+}
+
+fn get_os_process() -> String {
+    if cfg!(target_os = "windows") {
+        String::from("powershell.exe")
+    } else {
+        String::from("/bin/bash")
+    }
+}
--- a/crates/engine/Cargo.toml
+++ b/crates/engine/Cargo.toml
@ -22,4 +22,8 @@
    wasi-common                     = "0.30"
    wasi-experimental-http-wasmtime = "0.6.0"
    wasmtime                        = "0.30"
+    wasmtime-wasi                   = "0.30"
    witx-bindgen-rust               = { git = "https://github.com/bytecodealliance/witx-bindgen", rev = "0fea5183d121fdf736585ab9d291b3cc08487cd0" }
+
+[dev-dependencies]
+    witx-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/witx-bindgen", rev = "b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" }
--- a/crates/engine/src/lib.rs
+++ b/crates/engine/src/lib.rs
@ -2,9 +2,12 @@

 #![deny(missing_docs)]

+use anyhow::Result;
 use std::sync::Arc;
 use wasi_common::WasiCtx;
-use wasmtime::{Engine, InstancePre};
+use wasi_experimental_http_wasmtime::HttpCtx;
+use wasmtime::{Engine, Instance, InstancePre, Linker, Module, Store};
+use wasmtime_wasi::sync::{ambient_authority, Dir, WasiCtxBuilder};

 /// Engine configuration.
 #[derive(Clone, Default)]
@ -26,15 +29,26 @@ impl Config {
        preopen_dirs: Vec<(String, String)>,
        allowed_http_hosts: Option<Vec<String>>,
    ) -> Self {
+        Self {
+            env_vars,
+            preopen_dirs,
+            allowed_http_hosts,
+            ..Default::default()
+        }
+    }
+
+    /// Create a default instance of the configuration instance.
+    pub fn default() -> Self {
+        // In order for Wasmtime to run WebAssembly components, multi memory
+        // and module linking must always be enabled.
+        // See https://github.com/bytecodealliance/witx-bindgen/blob/main/crates/wasmlink
        let mut wasmtime_config = wasmtime::Config::default();
        wasmtime_config.wasm_multi_memory(true);
        wasmtime_config.wasm_module_linking(true);

        Self {
-            env_vars,
-            preopen_dirs,
-            allowed_http_hosts,
            wasmtime_config,
+            ..Default::default()
        }
    }
 }
@ -48,15 +62,139 @@ pub struct RuntimeContext<T> {
    pub data: Option<T>,
 }

-/// The generic execution context.
-#[derive(Clone)]
-pub struct ExecutionContext<T: Default> {
+/// Builder for the execution context.
+#[derive(Default)]
+pub struct ExecutionContextBuilder<T: Default> {
    /// Entrypoint of the WebAssembly compnent.
    pub entrypoint_path: String,
    /// Top-level runtime configuration.
    pub config: Config,
-    /// Pre-initialized WebAssembly instance.
-    pub pre: Arc<InstancePre<RuntimeContext<T>>>,
+    /// Linker used to configure the execution context.
+    pub linker: Linker<RuntimeContext<T>>,
+    /// Store used to configure the execution context.
+    pub store: Store<RuntimeContext<T>>,
    /// Wasmtime engine.
    pub engine: Engine,
 }
+
+impl<T: Default> ExecutionContextBuilder<T> {
+    /// Create a new instance of the execution builder.
+    pub fn new(entrypoint_path: String, config: Config) -> Result<ExecutionContextBuilder<T>> {
+        let data = RuntimeContext::default();
+        let engine = Engine::new(&config.wasmtime_config)?;
+        let store = Store::new(&engine, data);
+        let linker = Linker::new(&engine);
+
+        Ok(Self {
+            entrypoint_path,
+            config,
+            linker,
+            store,
+            engine,
+        })
+    }
+
+    /// Configure the WASI linker imports for the current execution context.
+    pub fn link_wasi<'a>(&'a mut self) -> Result<&'a Self> {
+        wasmtime_wasi::add_to_linker(&mut self.linker, |ctx| ctx.wasi.as_mut().unwrap())?;
+        Ok(self)
+    }
+
+    /// Configure the HTTP linker imports for the current execution context.
+    pub fn link_http<'a>(&'a mut self) -> Result<&'a Self> {
+        let hosts = &self.config.allowed_http_hosts.clone();
+        let http = HttpCtx::new(hosts.clone(), None)?;
+        http.add_to_linker(&mut self.linker)?;
+        Ok(self)
+    }
+
+    /// Build a new instance of the execution context by pre-instantiating the entrypoint module.
+    pub fn build(self) -> Result<ExecutionContext<T>> {
+        let module = Module::from_file(&self.engine, &self.entrypoint_path)?;
+        let pre = self.linker.instantiate_pre(self.store, &module)?;
+        Ok(ExecutionContext {
+            config: self.config,
+            engine: self.engine,
+            pre: Arc::new(pre),
+        })
+    }
+
+    /// Build a new default instance of the execution context by pre-instantiating the entrypoint module.
+    pub fn build_default(entrypoint_path: &str, config: Config) -> Result<ExecutionContext<T>> {
+        let mut builder = Self::new(entrypoint_path.into(), config)?;
+        builder.link_wasi()?;
+        builder.link_http()?;
+
+        builder.build()
+    }
+}
+
+/// The generic execution context.
+#[derive(Clone)]
+pub struct ExecutionContext<T: Default> {
+    /// Top-level runtime configuration.
+    pub config: Config,
+    /// Wasmtime engine.
+    pub engine: Engine,
+    /// Pre-initialized WebAssembly instance.
+    pub pre: Arc<InstancePre<RuntimeContext<T>>>,
+}
+
+impl<T: Default> ExecutionContext<T> {
+    /// Prepare an instance with actual data
+    pub fn prepare(&self, data: Option<T>) -> Result<(Store<RuntimeContext<T>>, Instance)> {
+        let mut store = self.make_store(data)?;
+        let instance = self.pre.instantiate(&mut store)?;
+        Ok((store, instance))
+    }
+
+    fn make_store(&self, data: Option<T>) -> Result<Store<RuntimeContext<T>>> {
+        let mut ctx = RuntimeContext::default();
+        ctx.data = data;
+        let mut wasi_ctx = WasiCtxBuilder::new()
+            .inherit_stdio()
+            .envs(&self.config.env_vars)?;
+
+        for (guest, host) in &self.config.preopen_dirs {
+            wasi_ctx =
+                wasi_ctx.preopened_dir(Dir::open_ambient_dir(host, ambient_authority())?, guest)?;
+        }
+
+        ctx.wasi = Some(wasi_ctx.build());
+
+        let store = Store::new(&self.engine, ctx);
+        Ok(store)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use anyhow::Result;
+    use echo::*;
+    use std::sync::Arc;
+
+    witx_bindgen_wasmtime::import!("crates/engine/tests/echo.witx");
+    const RUST_ENTRYPOINT_PATH: &str = "tests/rust-echo/target/wasm32-wasi/release/echo.wasm";
+
+    #[derive(Clone)]
+    pub struct EchoEngine(pub Arc<ExecutionContext<EchoData>>);
+
+    impl EchoEngine {
+        pub fn execute(&self, msg: &str) -> Result<String> {
+            let (mut store, instance) = self.0.prepare(None)?;
+            let e = Echo::new(&mut store, &instance, |host| host.data.as_mut().unwrap())?;
+            let res = e.echo(&mut store, msg)?;
+            Ok(res)
+        }
+    }
+
+    #[test]
+    fn test_rust_echo() {
+        let e = ExecutionContextBuilder::build_default(RUST_ENTRYPOINT_PATH, Config::default())
+            .unwrap();
+        let e = EchoEngine(Arc::new(e));
+
+        assert_eq!(e.execute("Fermyon").unwrap(), "Hello, Fermyon".to_string());
+    }
+}
--- a/crates/engine/tests/echo.witx
+++ b/crates/engine/tests/echo.witx
@ -0,0 +1 @@
+echo: function(msg: string) -> string
--- a/crates/engine/tests/rust-echo/Cargo.lock
+++ b/crates/engine/tests/rust-echo/Cargo.lock
@ -0,0 +1,140 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "anyhow"
+version = "1.0.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1"
+
+[[package]]
+name = "async-trait"
+version = "0.1.51"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "echo"
+version = "0.1.0"
+dependencies = [
+ "witx-bindgen-rust",
+]
+
+[[package]]
+name = "heck"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
+dependencies = [
+ "unicode-segmentation",
+]
+
+[[package]]
+name = "id-arena"
+version = "2.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba508cc11742c0dc5c1659771673afbab7a0efab23aa17e854cbab0837ed0b43"
+dependencies = [
+ "unicode-xid",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "syn"
+version = "1.0.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2afee18b8beb5a596ecb4a2dce128c719b4ba399d34126b9e4396e3f9860966"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
+
+[[package]]
+name = "witx-bindgen-gen-core"
+version = "0.1.0"
+source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e"
+dependencies = [
+ "anyhow",
+ "witx2",
+]
+
+[[package]]
+name = "witx-bindgen-gen-rust"
+version = "0.1.0"
+source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e"
+dependencies = [
+ "heck",
+ "witx-bindgen-gen-core",
+]
+
+[[package]]
+name = "witx-bindgen-gen-rust-wasm"
+version = "0.1.0"
+source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e"
+dependencies = [
+ "heck",
+ "witx-bindgen-gen-core",
+ "witx-bindgen-gen-rust",
+]
+
+[[package]]
+name = "witx-bindgen-rust"
+version = "0.1.0"
+source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e"
+dependencies = [
+ "async-trait",
+ "witx-bindgen-rust-impl",
+]
+
+[[package]]
+name = "witx-bindgen-rust-impl"
+version = "0.1.0"
+source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e"
+dependencies = [
+ "proc-macro2",
+ "syn",
+ "witx-bindgen-gen-core",
+ "witx-bindgen-gen-rust-wasm",
+]
+
+[[package]]
+name = "witx2"
+version = "0.1.0"
+source = "git+https://github.com/bytecodealliance/witx-bindgen?rev=b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e#b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e"
+dependencies = [
+ "anyhow",
+ "id-arena",
+]
--- a/crates/engine/tests/rust-echo/Cargo.toml
+++ b/crates/engine/tests/rust-echo/Cargo.toml
@ -0,0 +1,13 @@
+[package]
+    name    = "echo"
+    version = "0.1.0"
+    edition = "2021"
+    authors = ["Radu Matei <radu.matei@fermyon.com>"]
+
+[lib]
+    crate-type = ["cdylib"]
+
+[dependencies]
+    witx-bindgen-rust = { git = "https://github.com/bytecodealliance/witx-bindgen", rev = "b548ce6b35fe53feebb43c080dbe61e1ef4a5e8e" }
+
+[workspace]
--- a/crates/engine/tests/rust-echo/src/lib.rs
+++ b/crates/engine/tests/rust-echo/src/lib.rs
@ -0,0 +1,9 @@
+witx_bindgen_rust::export!("../echo.witx");
+
+struct Echo {}
+
+impl echo::Echo for Echo {
+    fn echo(msg: String) -> String {
+        format!("Hello, {}", msg)
+    }
+}