Add aarch64 support across CubeSandbox

CubeMaster/go.mod

@@ -5,7 +5,6 @@ go 1.24.8
 // toolchain go1.22.9
 
 require (
-	github.com/agiledragon/gomonkey v2.0.2+incompatible
 	github.com/agiledragon/gomonkey/v2 v2.9.0
 	github.com/alicebob/miniredis/v2 v2.35.0
 	github.com/fsnotify/fsnotify v1.9.0

CubeMaster/go.sum

@@ -6,8 +6,6 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/agiledragon/gomonkey v2.0.2+incompatible h1:eXKi9/piiC3cjJD1658mEE2o3NjkJ5vDLgYjCQu0Xlw=
-github.com/agiledragon/gomonkey v2.0.2+incompatible/go.mod h1:2NGfXu1a80LLr2cmWXGBDaHEjb1idR6+FVlX5T3D9hw=
 github.com/agiledragon/gomonkey/v2 v2.9.0 h1:PDiKKybR596O6FHW+RVSG0Z7uGCBNbmbUXh3uCNQ7Hc=
 github.com/agiledragon/gomonkey/v2 v2.9.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY=
 github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=

CubeMaster/integration/mock_init.go

@@ -18,7 +18,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/agiledragon/gomonkey"
+	"github.com/agiledragon/gomonkey/v2"
 	"github.com/alicebob/miniredis/v2"
 	"github.com/gomodule/redigo/redis"
 	"github.com/google/uuid"

CubeNet/cubevs/bpf_arm64.go

@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2026 Tencent Inc. All rights reserved.
+
+//go:build arm64
+
+package cubevs
+
+import (
+	"bytes"
+	_ "embed"
+	"fmt"
+
+	"github.com/cilium/ebpf"
+)
+
+// loadLocalgw returns the embedded CollectionSpec for localgw.
+func loadLocalgw() (*ebpf.CollectionSpec, error) {
+	reader := bytes.NewReader(_LocalgwBytes)
+	spec, err := ebpf.LoadCollectionSpecFromReader(reader)
+	if err != nil {
+		return nil, fmt.Errorf("can't load localgw: %w", err)
+	}
+	return spec, nil
+}
+
+// loadMvmtap returns the embedded CollectionSpec for mvmtap.
+func loadMvmtap() (*ebpf.CollectionSpec, error) {
+	reader := bytes.NewReader(_MvmtapBytes)
+	spec, err := ebpf.LoadCollectionSpecFromReader(reader)
+	if err != nil {
+		return nil, fmt.Errorf("can't load mvmtap: %w", err)
+	}
+	return spec, nil
+}
+
+// loadNodenic returns the embedded CollectionSpec for nodenic.
+func loadNodenic() (*ebpf.CollectionSpec, error) {
+	reader := bytes.NewReader(_NodenicBytes)
+	spec, err := ebpf.LoadCollectionSpecFromReader(reader)
+	if err != nil {
+		return nil, fmt.Errorf("can't load nodenic: %w", err)
+	}
+	return spec, nil
+}
+
+// Reuse the checked-in little-endian eBPF objects. The original bpf2go wrappers
+// are amd64-only, but the objects themselves are eBPF bytecode and are usable on
+// arm64 Linux as well.
+//
+//go:embed localgw_x86_bpfel.o
+var _LocalgwBytes []byte
+
+//go:embed mvmtap_x86_bpfel.o
+var _MvmtapBytes []byte
+
+//go:embed nodenic_x86_bpfel.o
+var _NodenicBytes []byte

CubeProxy/Dockerfile

@@ -1,4 +1,5 @@
-FROM cube-sandbox-image.tencentcloudcr.com/opensource/openresty:1.21.4.1-6-alpine-fat
+ARG BASE_IMAGE=cube-sandbox-image.tencentcloudcr.com/opensource/openresty:1.21.4.1-6-alpine-fat
+FROM ${BASE_IMAGE}
 
 LABEL maintainer="staryxchen <staryxchen@tencent.com>"
 

CubeShim/shim/src/common/utils.rs

@@ -47,8 +47,35 @@ impl Utils {
     pub fn load_spec(bundle: &str) -> CResult<Spec> {
         let mut conf_path = PathBuf::from(bundle);
         conf_path.push("config.json");
-        let spec = Spec::load(conf_path)
-            .map_err(|e| format!("load config failed:{} bundle:{}", e, bundle))?;
+
+        // Read the config.json as a raw JSON value first, so we can sanitize
+        // fields that the oci-spec 0.6.x deserializer rejects. Recent
+        // containerd releases emit `linux.seccomp = {"defaultAction": ""}`
+        // when the runtime spec has no seccomp profile, and the empty string
+        // fails to deserialize into the LinuxSeccompAction enum, which makes
+        // sandbox creation fall over with an opaque serde error.
+        //
+        // Strip the empty-action seccomp object up-front; an absent seccomp
+        // section is the correct representation of "no profile applied".
+        let data = fs::read_to_string(&conf_path).map_err(|e| {
+            format!(
+                "load config failed:read file {:?}: {} bundle:{}",
+                conf_path, e, bundle
+            )
+        })?;
+        let mut value: serde_json::Value = serde_json::from_str(&data)
+            .map_err(|e| format!("load config failed:parse json: {} bundle:{}", e, bundle))?;
+        if let Some(linux) = value.get_mut("linux") {
+            if let Some(seccomp) = linux.get("seccomp") {
+                if seccomp.get("defaultAction").and_then(|v| v.as_str()) == Some("") {
+                    if let Some(obj) = linux.as_object_mut() {
+                        obj.remove("seccomp");
+                    }
+                }
+            }
+        }
+        let spec: Spec = serde_json::from_value(value)
+            .map_err(|e| format!("load config failed:serde failed:{} bundle:{}", e, bundle))?;
         Ok(spec)
     }
 

CubeShim/shim/src/hypervisor/config.rs

@@ -70,24 +70,36 @@ pub struct VmConfig {
 
 impl Default for VmConfig {
     fn default() -> Self {
-        let params = vec![
+        let mut params = vec![
             "root=/dev/pmem0".to_string(),
             "rootflags=dax,errors=remount-ro ro".to_string(),
             "rootfstype=ext4".to_string(),
             "panic=1".to_string(),
-            "no_timer_check".to_string(),
-            "noreplace-smp".to_string(),
-            "printk.devkmsg=on".to_string(),
-            "console=hvc0".to_string(),
+        ];
+        #[cfg(target_arch = "x86_64")]
+        params.extend(["no_timer_check".to_string(), "noreplace-smp".to_string()]);
+        params.extend(["printk.devkmsg=on".to_string()]);
+        #[cfg(target_arch = "aarch64")]
+        params.push("console=ttyAMA0,115200".to_string());
+        #[cfg(not(target_arch = "aarch64"))]
+        params.push("console=hvc0".to_string());
+        params.extend([
             "net.ifnames=0".to_string(),
             "audit=0".to_string(),
             "LANG=C".to_string(),
             "raid=noautodetect".to_string(),
-            "earlyprintk=ttyS0".to_string(),
             "agent.debug_console".to_string(),
             "agent.debug_console_vport=1026".to_string(),
+        ]);
+        // x86-only kernel cmdline parameters. The ARM kernel does not recognize
+        // these and forwards unknown tokens as argv to /sbin/init (= cube-agent),
+        // making clap's argument parser fail and the agent exit immediately
+        // (kernel panic on init / "unrecognized argument" from agent).
+        #[cfg(target_arch = "x86_64")]
+        params.extend([
+            "earlyprintk=ttyS0".to_string(),
             "mitigations=off".to_string(),
-        ];
+        ]);
 
         let pmems = vec![PmemConfig {
             file: PathBuf::from(IMAGE_PATH),
@@ -452,24 +464,32 @@ mod tests {
         assert_eq!(config.console.mode, ConsoleOutputMode::Tty);
         assert_eq!(config.rng.src, PathBuf::from("/dev/urandom"));
 
-        let params = vec![
+        let mut params = vec![
             "root=/dev/pmem0".to_string(),
             "rootflags=dax,errors=remount-ro ro".to_string(),
             "rootfstype=ext4".to_string(),
             "panic=1".to_string(),
-            "no_timer_check".to_string(),
-            "noreplace-smp".to_string(),
-            "printk.devkmsg=on".to_string(),
-            "console=hvc0".to_string(),
+        ];
+        #[cfg(target_arch = "x86_64")]
+        params.extend(["no_timer_check".to_string(), "noreplace-smp".to_string()]);
+        params.extend(["printk.devkmsg=on".to_string()]);
+        #[cfg(target_arch = "aarch64")]
+        params.push("console=ttyAMA0,115200".to_string());
+        #[cfg(not(target_arch = "aarch64"))]
+        params.push("console=hvc0".to_string());
+        params.extend([
             "net.ifnames=0".to_string(),
             "audit=0".to_string(),
             "LANG=C".to_string(),
             "raid=noautodetect".to_string(),
-            "earlyprintk=ttyS0".to_string(),
             "agent.debug_console".to_string(),
             "agent.debug_console_vport=1026".to_string(),
+        ]);
+        #[cfg(target_arch = "x86_64")]
+        params.extend([
+            "earlyprintk=ttyS0".to_string(),
             "mitigations=off".to_string(),
-        ];
+        ]);
         assert_eq!(config.cmdlines, params);
     }
 

CubeShim/shim/src/hypervisor/cube_hypervisor.rs

@@ -76,7 +76,10 @@ impl CubeHypervisor {
             return Err(self.status_err("oops: ch is not None".to_string()));
         }
         cube_hypervisor::set_runtime_seccomp_rules(vec![
+            #[cfg(target_arch = "x86_64")]
             (libc::SYS_mkdir, vec![]),
+            #[cfg(target_arch = "aarch64")]
+            (libc::SYS_mkdirat, vec![]),
             (libc::SYS_getsockopt, vec![]),
             (libc::SYS_setsockopt, vec![]),
             (libc::SYS_faccessat2, vec![]),

CubeShim/shim/src/sandbox/sb.rs

@@ -43,7 +43,7 @@ use tokio::time::{sleep, Duration};
 
 use super::device;
 use super::disk::Disk;
-use super::pmem::{self, Pmem};
+use super::pmem::Pmem;
 //use tokio_uring::fs::UnixStream;
 
 const ANNO_SANDBOX_DNS: &str = "cube.sandbox.dns";
@@ -776,23 +776,131 @@ impl SandBox {
         {
             let ch = self.ch.as_mut().unwrap().lock().await;
             let start = Instant::now();
-            let ev = ch
-                .wait_notify(Duration::from_nanos(1000 * 1000 * 1000 * 10 as u64))
-                .await?;
+            let total_wait = Duration::from_secs(30);
+            let wait_res = ch.wait_notify(total_wait).await;
+            drop(ch);
 
-            if CH::NotifyEvent::VsockServerReady != ev {
-                return Err(format!(
-                    "Not an expected event, expected:{:?}, actual:{:?}",
-                    CH::NotifyEvent::VsockServerReady,
+            match &wait_res {
+                Ok(ev) if *ev == CH::NotifyEvent::VsockServerReady => {
+                    infof!(
+                        self.log,
+                        "vm ready, vsock is listening (notify path), cost:{}",
+                        start.elapsed().as_millis()
+                    );
+                    return Ok(snapshot);
+                }
+                Ok(ev) if *ev == CH::NotifyEvent::VmShutdown => {
+                    return Err(format!(
+                        "Not an expected event, expected:{:?}, actual:{:?}",
+                        CH::NotifyEvent::VsockServerReady,
+                        ev
+                    ));
+                }
+                Ok(ev) => warnf!(
+                    self.log,
+                    "wait_notify got unexpected non-fatal event {:?}, fallback to vsock probe",
                     ev
-                ));
+                ),
+                Err(e) => warnf!(
+                    self.log,
+                    "wait_notify VsockServerReady failed: {}, fallback to vsock probe",
+                    e
+                ),
             }
-            let duration = start.elapsed().as_millis();
-            infof!(self.log, "vm ready, vsock is listening, cost:{}", duration);
+
+            let probe_budget = Duration::from_secs(15);
+            Self::probe_vsock_ready(&self.id, probe_budget, &self.log)
+                .await
+                .map_err(|e| {
+                    let orig = match &wait_res {
+                        Ok(ev) => format!("{:?}", ev),
+                        Err(orig) => orig.clone(),
+                    };
+                    format!(
+                        "vsock not ready: notify={}, probe={}",
+                        orig, e
+                    )
+                })?;
+            infof!(
+                self.log,
+                "vm ready, vsock is listening (probe path), cost:{}",
+                start.elapsed().as_millis()
+            );
         }
         Ok(snapshot)
     }
 
+    async fn probe_vsock_ready(
+        sandbox_id: &str,
+        budget: Duration,
+        log: &Log,
+    ) -> std::result::Result<(), String> {
+        let vsock_path = Utils::vsock_path(sandbox_id);
+        const AGENT_VSOCK_PORT: u32 = 1024;
+        const PROBE_INTERVAL: Duration = Duration::from_millis(200);
+        let per_try_timeout = std::cmp::min(budget, Duration::from_millis(2000));
+
+        let deadline = std::time::Instant::now() + budget;
+        let mut last_err = String::from("not attempted");
+        while std::time::Instant::now() < deadline {
+            if !vsock_path.exists() {
+                last_err = format!("vsock socket {:?} not bound yet", vsock_path);
+            } else {
+                match tokio::time::timeout(
+                    per_try_timeout,
+                    Self::try_hybrid_vsock_handshake(&vsock_path, AGENT_VSOCK_PORT),
+                )
+                .await
+                {
+                    Ok(Ok(())) => return Ok(()),
+                    Ok(Err(e)) => last_err = format!("handshake: {}", e),
+                    Err(_) => {
+                        last_err = format!(
+                            "handshake timeout {}ms",
+                            per_try_timeout.as_millis()
+                        )
+                    }
+                }
+            }
+            debugf!(log, "vsock probe: {}", last_err);
+            sleep(PROBE_INTERVAL).await;
+        }
+        Err(format!(
+            "vsock probe gave up in {}ms, last: {}",
+            budget.as_millis(),
+            last_err
+        ))
+    }
+
+    async fn try_hybrid_vsock_handshake(
+        vsock_path: &PathBuf,
+        port: u32,
+    ) -> std::result::Result<(), String> {
+        use tokio::io::{AsyncBufReadExt, AsyncWriteExt};
+        use tokio::net::UnixStream;
+
+        let mut conn = UnixStream::connect(vsock_path)
+            .await
+            .map_err(|e| format!("connect {:?} failed: {}", vsock_path, e))?;
+        conn.write_all(format!("CONNECT {}\n", port).as_bytes())
+            .await
+            .map_err(|e| format!("write CONNECT failed: {}", e))?;
+        let mut reader = tokio::io::BufReader::new(conn);
+        let mut response = String::new();
+        let n = reader
+            .read_line(&mut response)
+            .await
+            .map_err(|e| format!("read response failed: {}", e))?;
+        if n == 0 {
+            return Err("EOF before response (muxer/guest closed; likely no listener on guest:1024 -> agent ttRPC server not started)".to_string());
+        }
+        if response.contains("OK") {
+            Ok(())
+        } else {
+            Err(format!("handshake response: {:?}", response.trim()))
+        }
+    }
+
     async fn boot_vm(&mut self) -> CResult<()> {
         let config = self.prepare_resource().await?;
         let mut ch = self.ch.as_mut().unwrap().lock().await;

CubeShim/shim/src/snapshot/mod.rs

@@ -205,7 +205,10 @@ impl Snapshot {
     fn launch_vmm(&mut self) -> CResult<()> {
         //launch
         cube_hypervisor::set_runtime_seccomp_rules(vec![
+            #[cfg(target_arch = "x86_64")]
             (libc::SYS_mkdir, vec![]),
+            #[cfg(target_arch = "aarch64")]
+            (libc::SYS_mkdirat, vec![]),
             (libc::SYS_getsockopt, vec![]),
             (libc::SYS_setsockopt, vec![]),
         ]);