Support for Apple M1. (#37)

Release pipeline prepares arm64-darwin gem.

Alter rake tasks so that macOS can cross-compile arm64 or x86_64. We need to support cross-compile because we don't have access to arm64 macOS cloud machines.

Author: graebm

.github/workflows/ci.yml

@@ -88,3 +88,20 @@ jobs:
           python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
           chmod a+x builder
           ./builder build -p ${{ env.PACKAGE_NAME }} downstream
+
+  # Cross-compile for Apple silicon.
+  # It would be better to run tests natively on one of these machines, but we
+  # don't currently have access to one in the cloud.
+  osx-arm64-cross-compile:
+    runs-on: macos-latest
+    steps:
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.7
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - name: Build ${{ env.PACKAGE_NAME }} + consumers
+        run: |
+          bundle install
+          bundle exec rake "gem:aws-crt:platform[arm64]"

.github/workflows/release.yml

@@ -3,8 +3,7 @@ name: Release
 on:
   push:
     branches:
-      - '*'  # TODO: Remove after testing...
-      - 'master'
+      - 'main'
 
 env:
   BUILDER_VERSION: v0.8.19
@@ -78,25 +77,29 @@ jobs:
       matrix:
         os: [macos]
         ruby: [2.7]
-        arch: [x64]
+        arch: [x86_64, arm64]
     runs-on: macos-latest
     steps:
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
+
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
       - name: Generate Gem artifacts
         run: |
-          python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
-          chmod a+x builder
-          ./builder build -p ${{ env.PACKAGE_NAME }} downstream
+          bundle install
+          bundle exec rake "gem:aws-crt:platform[${{ matrix.arch }}]"
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v2
         with:
           name: out_${{ matrix.os }}_${{ matrix.arch }}
           path: |
-            aws-crt-ruby/gems/aws-crt/bin/
-            aws-crt-ruby/gems/aws-crt/pkg/
+            gems/aws-crt/bin/
+            gems/aws-crt/pkg/
 
   package:
     needs: [windows, osx, linux]
@@ -115,10 +118,15 @@ jobs:
         with:
           name: out_windows_x64
 
-      - name: Download OSX
+      - name: Download OSX x64
+        uses: actions/download-artifact@v2
+        with:
+          name: out_macos_x86_64
+
+      - name: Download OSX arm64
         uses: actions/download-artifact@v2
         with:
-          name: out_macos_x64
+          name: out_macos_arm64
 
       - name: Download Linux x64
         uses: actions/download-artifact@v2

CONTRIBUTING.md

@@ -23,7 +23,7 @@ reported the issue. Please try to include as much information as you can. Detail
 ## Contributing via Pull Requests
 Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that:
 
-1. You are working against the latest source on the *master* branch.
+1. You are working against the latest source on the *main* branch.
 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already.
 3. You open an issue to discuss any significant work - we would hate for your time to be wasted.
 

gems/aws-crt/ext/compile.rb

@@ -41,15 +41,16 @@ def find_file(name, search_dirs, base_dir)
 end
 
 # Compile bin to expected location
-def compile_bin
-  platform = local_platform
+def compile_bin(cpu = host_cpu)
+  platform = target_platform(cpu)
   native_dir = File.expand_path('../aws-crt-ffi', File.dirname(__FILE__))
-  tmp_build_dir = File.expand_path('../tmp/build', File.dirname(__FILE__))
+  tmp_dir = File.expand_path("../tmp/#{platform.cpu}", File.dirname(__FILE__))
+  tmp_build_dir = File.expand_path('build', tmp_dir)
 
   # We need cmake to "install" aws-crt-ffi so that the binaries end up in a
   # predictable location. But cmake still adds subdirectories we don't want,
   # so we'll "install" under tmp, and manually copy to bin/ after that.
-  tmp_install_dir = File.expand_path('../tmp/install', File.dirname(__FILE__))
+  tmp_install_dir = File.expand_path('install', tmp_dir)
 
   build_type = 'RelWithDebInfo'
 
@@ -59,8 +60,15 @@ def compile_bin
     "-B#{tmp_build_dir}",
     "-DCMAKE_INSTALL_PREFIX=#{tmp_install_dir}",
     "-DCMAKE_BUILD_TYPE=#{build_type}",
+    '-DBUILD_TESTING=OFF',
   ]
 
+  # macOS can cross-compile for arm64 or x86_64.
+  # This lets us prepare both types of gems from either type of machine.
+  if platform.os == 'darwin'
+    config_cmd.append("-DCMAKE_OSX_ARCHITECTURES=#{platform.cpu}")
+  end
+
   build_cmd = [
     CMAKE,
     '--build', tmp_build_dir,
@@ -87,5 +95,5 @@ def compile_bin
     'lib', # some unix variants
   ]
   tmp_path = find_file(bin_name, search_dirs, tmp_install_dir)
-  FileUtils.cp(tmp_path, bin_dir)
+  FileUtils.cp(tmp_path, bin_dir, verbose: true)
 end

gems/aws-crt/lib/aws-crt/platforms.rb

@@ -9,13 +9,17 @@
 
 DEFAULT_BINARY = 'libaws-crt-ffi.so'
 
-# @return [String] returns Gem::Platform style name for the current system
-# similar to Gem::Platform.local but will return systems host os/cpu
-# for Jruby
+# @return [Gem::Platform] similar to Gem::Platform.local but will return
+# host os/cpu for Jruby
 def local_platform
   Gem::Platform.new(host_string)
 end
 
+# @return [Gem::Platform] return Gem::Platform for host os with target cpu
+def target_platform(cpu)
+  Gem::Platform.new(target_string(cpu))
+end
+
 # @return [String] return the file name for the CRT library for the platform
 def crt_bin_name(platform)
   OS_BINARIES[platform.os] || DEFAULT_BINARY
@@ -31,9 +35,14 @@ def crt_bin_path(platform)
   File.expand_path(crt_bin_name(platform), crt_bin_dir(platform))
 end
 
-# @return [String] generate a string that be used with Gem::Platform
+# @return [String] generate a string that can be used with Gem::Platform
 def host_string
-  "#{host_cpu}-#{host_os}"
+  target_string(host_cpu)
+end
+
+# @return [String] generate a string that can be used with Gem::Platform
+def target_string(cpu)
+  "#{cpu}-#{host_os}"
 end
 
 # @return [String] host cpu, even on jruby

gems/aws-crt/tasks/compile.rake

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
 
-desc 'Compile CRT library and move to bin'
-task :bin do
+desc 'Compile CRT library and move to bin. Specify [cpu] to cross-compile.'
+task :bin, [:cpu] do |_, args|
+  require_relative '../lib/aws-crt/platforms'
+  args.with_defaults(:cpu => host_cpu)
+
   require_relative '../ext/compile'
-  compile_bin
+  compile_bin(args[:cpu])
 end
 
 desc 'Copies all binaries from the top level bin directory'

gems/aws-crt/tasks/gem.rake

@@ -1,13 +1,15 @@
 # frozen_string_literal: true
 
-desc 'Build the aws-crt gem for the local platform'
-task 'gem:aws-crt:local' => [:bin] do
+desc 'Build aws-crt platform gem'
+task 'gem:aws-crt:platform', [:cpu] => [:bin] do |_, args|
   require 'rubygems/package'
   require 'fileutils'
   require_relative '../lib/aws-crt/platforms'
 
+  args.with_defaults(:cpu => host_cpu)
+
   FileUtils.chdir('gems/aws-crt') do
-    platform = local_platform
+    platform = target_platform(args[:cpu])
     binary_name = crt_bin_name(platform)
 
     FileUtils.mkdir_p('pkg/', verbose: true)
@@ -27,7 +29,7 @@ task 'gem:aws-crt:local' => [:bin] do
 end
 
 desc 'Build the aws-crt gem for the local platform'
-task 'gem:aws-crt' => 'gem:aws-crt:local'
+task 'gem:aws-crt' => 'gem:aws-crt:platform'
 
 desc 'Build the aws-crt gem for pure-ruby'
 task 'gem:aws-crt:pure-ruby' => :clean do