Reuse Ruby builds when static gem extensions are unchanged

Author: ledsun

CONTRIBUTING.md

@@ -34,6 +34,12 @@ $ rake npm:ruby-head-wasm-wasip2:check
 
 If you need to re-build Ruby, please clean `./rubies` directory, and run `rake npm:ruby-head-wasm-wasi` again.
 
+`rake npm:ruby-head-wasm-wasi` reuses Ruby unless its statically linked gem
+extensions change.
+
+After changing the compiler or build settings, also discard intermediate
+objects with `rake npm:ruby-head-wasm-wasi:clean-build`.
+
 ### npm package test structure
 
 The `ruby-wasm-wasi` package contains the shared JavaScript bindings and test suites, but it does not contain a Ruby WebAssembly binary. Version-specific packages provide the binary and invoke the shared tests with `RUBY_NPM_PACKAGE_ROOT`:

lib/ruby_wasm/packager/core.rb

@@ -1,5 +1,6 @@
 require "forwardable"
 require "pathname"
+require_relative "extension_cache"
 
 class RubyWasm::Packager::Core
   def initialize(packager)
@@ -281,22 +282,23 @@ def name
   class StaticLinking < BuildStrategy
     def build(executor, options)
       build = derive_build
+      cache =
+        RubyWasm::Packager::ExtensionCache.new(
+          build.crossruby.artifact,
+          build.crossruby.user_exts
+        )
       force_rebuild =
         options[:remake] || options[:clean] || options[:reconfigure]
-      if File.exist?(build.crossruby.artifact) && !force_rebuild
-        return build.crossruby.artifact
-      end
-      build.crossruby.clean(executor) if options[:clean]
-
-      self.with_unbundled_env do
-        build.crossruby.build(
-          executor,
-          remake: options[:remake],
-          reconfigure: options[:reconfigure]
-        )
+      cache.fetch(force: force_rebuild) do
+        build.crossruby.clean(executor) if options[:clean]
+        with_unbundled_env do
+          build.crossruby.build(
+            executor,
+            remake: true,
+            reconfigure: options[:reconfigure]
+          )
+        end
       end
-
-      build.crossruby.artifact
     end
 
     def cache_key(digest)

lib/ruby_wasm/packager/extension_cache.rb

@@ -0,0 +1,45 @@
+require "digest"
+require "fileutils"
+
+class RubyWasm::Packager::ExtensionCache
+  def initialize(artifact, extensions)
+    @artifact = artifact
+    @extensions = extensions
+    @record = "#{artifact}.extensions.sha256"
+  end
+
+  # Inputs may be edited during a build, so only record builds whose inputs
+  # stayed unchanged. Invalidate the previous record first so a failed forced
+  # rebuild cannot leave an older artifact eligible for reuse.
+  def fetch(force: false)
+    extensions_key_before_build = key
+    if !force && reusable?(extensions_key_before_build)
+      RubyWasm.logger.info "Reusing Ruby: #{@artifact}"
+      return @artifact
+    end
+
+    FileUtils.rm_f(@record)
+    yield
+
+    extensions_key_after_build = key
+    if extensions_key_before_build == extensions_key_after_build
+      File.write(@record, extensions_key_after_build)
+    end
+    @artifact
+  end
+
+  private
+
+  def reusable?(key)
+    return false unless File.file?(@artifact)
+    return true if @extensions.empty?
+
+    File.file?(@record) && File.read(@record) == key
+  end
+
+  def key
+    digest = Digest::SHA256.new
+    @extensions.each { |ext| ext.cache_key(digest) }
+    digest.hexdigest
+  end
+end

rakelib/packaging.rake

@@ -105,7 +105,7 @@ def build_ruby(pkg, base_dir, pkg_dir, binaryen, clean: false)
         sh env,
            "bundle", "exec",
            *build_command,
-           "--no-stdlib", "--remake",
+           "--no-stdlib",
            *(clean ? ["--clean"] : []),
            "-o",
            File.join(dist_dir, "ruby.wasm")

sig/ruby_wasm/packager.rbs

@@ -94,6 +94,16 @@ class RubyWasm::Packager
     end
   end
 
+  class ExtensionCache
+    @artifact: String
+    @extensions: Array[RubyWasm::CrossRubyExtProduct]
+    @record: String
+    def initialize: (String artifact, Array[RubyWasm::CrossRubyExtProduct] extensions) -> void
+    def fetch: (?force: bool?) { () -> void } -> String
+    private def reusable?: (String key) -> bool
+    private def key: () -> String
+  end
+
   class FileSystem
     @dest_dir: string
     @packager: RubyWasm::Packager

test/test_static_extension_cache.rb

@@ -0,0 +1,134 @@
+require "test-unit"
+require "tmpdir"
+require_relative "../lib/ruby_wasm"
+
+class StaticExtensionCacheTest < Test::Unit::TestCase
+  class CrossRuby
+    attr_reader :artifact, :calls, :extension, :user_exts, :clean_count
+    attr_accessor :fail_build, :during_build
+
+    def initialize(dir)
+      @artifact = File.join(dir, "ruby.tar.gz")
+      @extension = File.join(dir, "extension")
+      FileUtils.mkdir_p(@extension)
+      File.write(File.join(@extension, "js.c"), "original")
+      @user_exts = [
+        RubyWasm::CrossRubyExtProduct.new(@extension, nil, features: nil)
+      ]
+      @calls = []
+      @clean_count = 0
+    end
+
+    def build(_executor, **options)
+      @calls << options
+      @during_build.call if @during_build
+      File.write(artifact, "ruby")
+      raise "build failed" if fail_build
+    end
+
+    def clean(_executor)
+      @clean_count += 1
+      FileUtils.rm_f(artifact)
+    end
+  end
+
+  class Strategy < RubyWasm::Packager::Core::StaticLinking
+    def initialize(build)
+      @build = build
+    end
+
+    def derive_build
+      @build
+    end
+  end
+
+  def setup
+    @dir = Dir.mktmpdir
+    @build = Struct.new(:crossruby).new(CrossRuby.new(@dir))
+    @strategy = Strategy.new(@build)
+    @executor = Object.new
+  end
+
+  def teardown
+    FileUtils.remove_entry(@dir)
+  end
+
+  def build(**options)
+    @strategy.build(@executor, options)
+  end
+
+  def test_unchanged_extensions_reuse_artifact
+    build
+    build
+    assert_equal 1, @build.crossruby.calls.size
+  end
+
+  def test_extension_content_change_rebuilds
+    build
+    path = File.join(@build.crossruby.extension, "js.c")
+    File.write(path, "modified")
+    build
+    assert_equal 2, @build.crossruby.calls.size
+    assert_equal(
+      { remake: true, reconfigure: nil },
+      @build.crossruby.calls.last
+    )
+    assert_equal 0, @build.crossruby.clean_count
+  end
+
+  def test_added_and_deleted_extension_inputs_rebuild
+    build
+    path = File.join(@build.crossruby.extension, "extra.h")
+    File.write(path, "header")
+    build
+    FileUtils.rm_f(path)
+    build
+    assert_equal 3, @build.crossruby.calls.size
+  end
+
+  def test_existing_artifact_without_record_rebuilds
+    File.write(@build.crossruby.artifact, "old build")
+    build
+    assert_equal 1, @build.crossruby.calls.size
+  end
+
+  def test_existing_artifact_without_extensions_is_reused
+    @build.crossruby.user_exts.clear
+    File.write(@build.crossruby.artifact, "prebuilt Ruby")
+    build
+    assert_empty @build.crossruby.calls
+  end
+
+  def test_force_options_rebuild_even_with_matching_inputs
+    build
+    build(remake: true)
+    build(reconfigure: true)
+    build(clean: true)
+    assert_equal 4, @build.crossruby.calls.count { |call| call.is_a?(Hash) }
+    assert_equal 1, @build.crossruby.clean_count
+    assert_equal true, @build.crossruby.calls[2][:reconfigure]
+  end
+
+  def test_failed_forced_build_invalidates_previous_record
+    build
+    @build.crossruby.fail_build = true
+    assert_raise(RuntimeError) { build(remake: true) }
+    @build.crossruby.fail_build = false
+    build
+    assert_equal 3, @build.crossruby.calls.size
+  end
+
+  def test_inputs_edited_during_build_are_not_recorded
+    @build.crossruby.during_build =
+      proc do
+        File.write(
+          File.join(@build.crossruby.extension, "js.c"),
+          "edited during build"
+        )
+      end
+    build
+    @build.crossruby.during_build = nil
+    build
+    assert_equal 2, @build.crossruby.calls.size
+  end
+end