initial implementation of macOS crypto backend

eng/_util/buildutil/buildutil.go

@@ -91,6 +91,7 @@ func AppendExperimentEnv(experiment string) {
 	if strings.Contains(experiment, "opensslcrypto") ||
 		strings.Contains(experiment, "cngcrypto") ||
 		strings.Contains(experiment, "boringcrypto") ||
+		strings.Contains(experiment, "darwincrypto") ||
 		strings.Contains(experiment, "systemcrypto") {
 
 		experiment += ",allowcryptofallback"

eng/_util/cmd/run-builder/run-builder.go

@@ -166,11 +166,18 @@ func main() {
 			env("GO_BUILDER_NAME", goos+"-"+goarch)
 		}
 
-		cmdline := []string{
-			// Use the dist test command directly, because 'src/run.bash' isn't compatible with
-			// longtest. 'src/run.bash' sets 'GOPATH=/nonexist-gopath', which breaks modconv tests
-			// that download modules.
-			"go/bin/go", "tool", "dist", "test",
+		var cmdline []string
+		if goos == "darwin" {
+			cmdline = []string{
+				"GOROOT=$(go/bin/go env GOROOT)", "go/bin/go", "run", "cmd/dist", "test",
+			}
+		} else {
+			cmdline = []string{
+				// Use the dist test command directly, because 'src/run.bash' isn't compatible with
+				// longtest. 'src/run.bash' sets 'GOPATH=/nonexist-gopath', which breaks modconv tests
+				// that download modules.
+				"go/bin/go", "tool", "dist", "test",
+			}
 		}
 
 		if goos == "linux" {

eng/pipeline/stages/go-builder-matrix-stages.yml

@@ -81,6 +81,9 @@ stages:
             - { os: linux, arch: arm64, config: buildandpack }
         - ${{ if parameters.innerloop }}:
           - { os: darwin, arch: amd64, config: devscript }
+          - { os: darwin, arch: amd64, config: test }
+          - { experiment: darwincrypto, os: darwin, arch: amd64, config: test }
+          - { experiment: darwincrypto, os: darwin, arch: amd64, config: test, fips: true }
           - { os: linux, arch: amd64, config: devscript }
           - { os: linux, arch: amd64, config: test }
           - { os: linux, arch: amd64, config: test, distro: ubuntu }

patches/0001-Add-systemcrypto-GOEXPERIMENT.patch

@@ -11,18 +11,18 @@ information about the behavior.
 Includes new tests in "build_test.go" and "buildbackend_test.go" to help
 maintain this feature. For more information, see the test files.
 ---
- src/cmd/go/internal/modindex/build.go         | 54 ++++++++++++++
- src/cmd/go/internal/modindex/build_test.go    | 73 +++++++++++++++++++
- src/go/build/build.go                         | 54 ++++++++++++++
- src/go/build/buildbackend_test.go             | 66 +++++++++++++++++
+ src/cmd/go/internal/modindex/build.go         | 57 +++++++++++++
+ src/cmd/go/internal/modindex/build_test.go    | 73 ++++++++++++++++
+ src/go/build/build.go                         | 57 +++++++++++++
+ src/go/build/buildbackend_test.go             | 84 +++++++++++++++++++
  .../testdata/backendtags_openssl/main.go      |  3 +
  .../testdata/backendtags_openssl/openssl.go   |  3 +
  .../build/testdata/backendtags_system/main.go |  3 +
  .../backendtags_system/systemcrypto.go        |  3 +
- .../goexperiment/exp_systemcrypto_off.go      |  9 +++
- .../goexperiment/exp_systemcrypto_on.go       |  9 +++
+ .../goexperiment/exp_systemcrypto_off.go      |  9 ++
+ .../goexperiment/exp_systemcrypto_on.go       |  9 ++
  src/internal/goexperiment/flags.go            | 15 ++++
- 11 files changed, 292 insertions(+)
+ 11 files changed, 316 insertions(+)
  create mode 100644 src/cmd/go/internal/modindex/build_test.go
  create mode 100644 src/go/build/buildbackend_test.go
  create mode 100644 src/go/build/testdata/backendtags_openssl/main.go
@@ -33,16 +33,17 @@ maintain this feature. For more information, see the test files.
  create mode 100644 src/internal/goexperiment/exp_systemcrypto_on.go
 
 diff --git a/src/cmd/go/internal/modindex/build.go b/src/cmd/go/internal/modindex/build.go
-index b57f2f6368f0fe..9ddde1ce9a2286 100644
+index b4dacb0f523a8d..4315c288d10cb3 100644
 --- a/src/cmd/go/internal/modindex/build.go
 +++ b/src/cmd/go/internal/modindex/build.go
-@@ -880,13 +880,67 @@ func (ctxt *Context) matchTag(name string, allTags map[string]bool) bool {
+@@ -886,13 +886,70 @@ func (ctxt *Context) matchTag(name string, allTags map[string]bool) bool {
  		name = "goexperiment.boringcrypto" // boringcrypto is an old name for goexperiment.boringcrypto
  	}
 
 +	const system = "goexperiment.systemcrypto"
 +	const openssl = "goexperiment.opensslcrypto"
 +	const cng = "goexperiment.cngcrypto"
++	const darwin = "goexperiment.darwincrypto"
 +	const boring = "goexperiment.boringcrypto"
 +	// Implement the SystemCrypto GOEXPERIMENT logic. This is done here rather
 +	// than during GOEXPERIMENT parsing so "-tags goexperiment.systemcrypto"
@@ -63,11 +64,12 @@ index b57f2f6368f0fe..9ddde1ce9a2286 100644
 +	satisfiedByAnyBackend := name == system
 +	satisfiedBySystemCrypto :=
 +		(ctxt.GOOS == "linux" && name == openssl) ||
-+			(ctxt.GOOS == "windows" && name == cng)
++			(ctxt.GOOS == "windows" && name == cng) ||
++			(ctxt.GOOS == "darwin" && name == darwin)
 +	satisfiedBy := func(tag string) bool {
 +		if satisfiedByAnyBackend {
 +			switch tag {
-+			case openssl, cng, boring:
++			case openssl, cng, darwin, boring:
 +				return true
 +			}
 +		}
@@ -81,6 +83,7 @@ index b57f2f6368f0fe..9ddde1ce9a2286 100644
 +		if satisfiedByAnyBackend {
 +			allTags[openssl] = true
 +			allTags[cng] = true
++			allTags[darwin] = true
 +			allTags[boring] = true
 +		}
 +		if satisfiedBySystemCrypto {
@@ -184,16 +187,17 @@ index 00000000000000..1756c5d027fee0
 +	}
 +}
 diff --git a/src/go/build/build.go b/src/go/build/build.go
-index dd6cdc903a21a8..48adcfed5cf3cb 100644
+index 9ffffda08a99b1..78fd536fa6a6d1 100644
 --- a/src/go/build/build.go
 +++ b/src/go/build/build.go
-@@ -1947,13 +1947,67 @@ func (ctxt *Context) matchTag(name string, allTags map[string]bool) bool {
+@@ -1984,13 +1984,70 @@ func (ctxt *Context) matchTag(name string, allTags map[string]bool) bool {
  		name = "goexperiment.boringcrypto" // boringcrypto is an old name for goexperiment.boringcrypto
  	}
 
 +	const system = "goexperiment.systemcrypto"
 +	const openssl = "goexperiment.opensslcrypto"
 +	const cng = "goexperiment.cngcrypto"
++	const darwin = "goexperiment.darwincrypto"
 +	const boring = "goexperiment.boringcrypto"
 +	// Implement the SystemCrypto GOEXPERIMENT logic. This is done here rather
 +	// than during GOEXPERIMENT parsing so "-tags goexperiment.systemcrypto"
@@ -214,11 +218,12 @@ index dd6cdc903a21a8..48adcfed5cf3cb 100644
 +	satisfiedByAnyBackend := name == system
 +	satisfiedBySystemCrypto :=
 +		(ctxt.GOOS == "linux" && name == openssl) ||
-+			(ctxt.GOOS == "windows" && name == cng)
++			(ctxt.GOOS == "windows" && name == cng) ||
++			(ctxt.GOOS == "darwin" && name == darwin)
 +	satisfiedBy := func(tag string) bool {
 +		if satisfiedByAnyBackend {
 +			switch tag {
-+			case openssl, cng, boring:
++			case openssl, cng, darwin, boring:
 +				return true
 +			}
 +		}
@@ -232,6 +237,7 @@ index dd6cdc903a21a8..48adcfed5cf3cb 100644
 +		if satisfiedByAnyBackend {
 +			allTags[openssl] = true
 +			allTags[cng] = true
++			allTags[darwin] = true
 +			allTags[boring] = true
 +		}
 +		if satisfiedBySystemCrypto {
@@ -257,10 +263,10 @@ index dd6cdc903a21a8..48adcfed5cf3cb 100644
  		}
 diff --git a/src/go/build/buildbackend_test.go b/src/go/build/buildbackend_test.go
 new file mode 100644
-index 00000000000000..a22abbb42e37c0
+index 00000000000000..aa3c5f1007ed79
 --- /dev/null
 +++ b/src/go/build/buildbackend_test.go
-@@ -0,0 +1,66 @@
+@@ -0,0 +1,84 @@
 +// Copyright 2023 The Go Authors. All rights reserved.
 +// Use of this source code is governed by a BSD-style
 +// license that can be found in the LICENSE file.
@@ -318,14 +324,32 @@ index 00000000000000..a22abbb42e37c0
 +	if err != nil {
 +		t.Fatal(err)
 +	}
-+	want = []string{"goexperiment.boringcrypto", "goexperiment.cngcrypto", "goexperiment.opensslcrypto", "goexperiment.systemcrypto"}
++	want = []string{"goexperiment.boringcrypto", "goexperiment.cngcrypto", "goexperiment.darwincrypto", "goexperiment.opensslcrypto", "goexperiment.systemcrypto"}
 +	if !reflect.DeepEqual(p.AllTags, want) {
 +		t.Errorf("AllTags = %v, want %v", p.AllTags, want)
 +	}
 +	wantFiles = []string{"main.go", "systemcrypto.go"}
 +	if !reflect.DeepEqual(p.GoFiles, wantFiles) {
 +		t.Errorf("GoFiles = %v, want %v", p.GoFiles, wantFiles)
 +	}
++
++	ctxt.GOARCH = "amd64"
++	ctxt.GOOS = "darwin"
++	ctxt.BuildTags = []string{"goexperiment.darwincrypto"}
++	p, err = ctxt.ImportDir("testdata/backendtags_openssl", 0)
++	if err != nil {
++		t.Fatal(err)
++	}
++	// Given the current GOOS (darwin), systemcrypto would not affect the
++	// decision, so we don't want it to be included in AllTags.
++	want = []string{"goexperiment.opensslcrypto"}
++	if !reflect.DeepEqual(p.AllTags, want) {
++		t.Errorf("AllTags = %v, want %v", p.AllTags, want)
++	}
++	wantFiles = []string{"main.go"}
++	if !reflect.DeepEqual(p.GoFiles, wantFiles) {
++		t.Errorf("GoFiles = %v, want %v", p.GoFiles, wantFiles)
++	}
 +}
 diff --git a/src/go/build/testdata/backendtags_openssl/main.go b/src/go/build/testdata/backendtags_openssl/main.go
 new file mode 100644
@@ -394,14 +418,14 @@ index 00000000000000..9c5b0bbc7b99dc
 +const SystemCrypto = true
 +const SystemCryptoInt = 1
 diff --git a/src/internal/goexperiment/flags.go b/src/internal/goexperiment/flags.go
-index ae3cbaf89fa5dd..de79140b2d4780 100644
+index 31b3d0315b64f8..de1dfa6e567a71 100644
 --- a/src/internal/goexperiment/flags.go
 +++ b/src/internal/goexperiment/flags.go
 @@ -60,6 +60,21 @@ type Flags struct {
  	StaticLockRanking bool
  	BoringCrypto      bool
 
-+	// SystemCrypto enables the OpenSSL or CNG crypto experiment depending on
++	// SystemCrypto enables the OpenSSL, CNG or Darwin crypto experiment depending on
 +	// which one is appropriate on the target GOOS.
 +	//
 +	// If SystemCrypto is enabled but no crypto experiment is appropriate on the

patches/0002-Add-crypto-backend-foundation.patch

@@ -590,7 +590,7 @@ index 039bd82ed21f9f..69a97d9bf250be 100644
  	panic("boringcrypto: not available")
 diff --git a/src/crypto/ed25519/boring.go b/src/crypto/ed25519/boring.go
 new file mode 100644
-index 00000000000000..3a7d7b76c8d8d7
+index 00000000000000..ff5c426b4fdaea
 --- /dev/null
 +++ b/src/crypto/ed25519/boring.go
 @@ -0,0 +1,71 @@
@@ -619,11 +619,11 @@ index 00000000000000..3a7d7b76c8d8d7
 +}
 +
 +type boringPub struct {
-+	key  *boring.PublicKeyEd25519
++	key  boring.PublicKeyEd25519
 +	orig [PublicKeySize]byte
 +}
 +
-+func boringPublicKey(pub PublicKey) (*boring.PublicKeyEd25519, error) {
++func boringPublicKey(pub PublicKey) (boring.PublicKeyEd25519, error) {
 +	// Use the pointer to the underlying pub array as key.
 +	p := unsafe.SliceData(pub)
 +	b := pubCache.Get(p)
@@ -635,19 +635,19 @@ index 00000000000000..3a7d7b76c8d8d7
 +	copy(b.orig[:], pub)
 +	key, err := boring.NewPublicKeyEd25119(b.orig[:])
 +	if err != nil {
-+		return nil, err
++		return key, err
 +	}
 +	b.key = key
 +	pubCache.Put(p, b)
 +	return key, nil
 +}
 +
 +type boringPriv struct {
-+	key  *boring.PrivateKeyEd25519
++	key  boring.PrivateKeyEd25519
 +	orig [PrivateKeySize]byte
 +}
 +
-+func boringPrivateKey(priv PrivateKey) (*boring.PrivateKeyEd25519, error) {
++func boringPrivateKey(priv PrivateKey) (boring.PrivateKeyEd25519, error) {
 +	// Use the pointer to the underlying priv array as key.
 +	p := unsafe.SliceData(priv)
 +	b := privCache.Get(p)
@@ -659,7 +659,7 @@ index 00000000000000..3a7d7b76c8d8d7
 +	copy(b.orig[:], priv)
 +	key, err := boring.NewPrivateKeyEd25119(b.orig[:])
 +	if err != nil {
-+		return nil, err
++		return key, err
 +	}
 +	b.key = key
 +	privCache.Put(p, b)
@@ -793,7 +793,7 @@ index c1f8ff784e4a5c..308d814ff6302b 100644
  		return errors.New("ed25519: expected opts.Hash zero (unhashed message, for standard Ed25519) or SHA-512 (for Ed25519ph)")
 diff --git a/src/crypto/ed25519/notboring.go b/src/crypto/ed25519/notboring.go
 new file mode 100644
-index 00000000000000..b0cdd44d81c753
+index 00000000000000..77b69a3be88183
 --- /dev/null
 +++ b/src/crypto/ed25519/notboring.go
 @@ -0,0 +1,16 @@
@@ -807,10 +807,10 @@ index 00000000000000..b0cdd44d81c753
 +
 +import boring "crypto/internal/backend"
 +
-+func boringPublicKey(PublicKey) (*boring.PublicKeyEd25519, error) {
++func boringPublicKey(PublicKey) (boring.PublicKeyEd25519, error) {
 +	panic("boringcrypto: not available")
 +}
-+func boringPrivateKey(PrivateKey) (*boring.PrivateKeyEd25519, error) {
++func boringPrivateKey(PrivateKey) (boring.PrivateKeyEd25519, error) {
 +	panic("boringcrypto: not available")
 +}
 diff --git a/src/crypto/hkdf/hkdf.go b/src/crypto/hkdf/hkdf.go
@@ -936,15 +936,15 @@ index 00000000000000..c2c06d3bff8c74
 +}
 diff --git a/src/crypto/internal/backend/bbig/big.go b/src/crypto/internal/backend/bbig/big.go
 new file mode 100644
-index 00000000000000..20251a290dc2e0
+index 00000000000000..ab3f30825dcfa1
 --- /dev/null
 +++ b/src/crypto/internal/backend/bbig/big.go
 @@ -0,0 +1,17 @@
 +// Copyright 2022 The Go Authors. All rights reserved.
 +// Use of this source code is governed by a BSD-style
 +// license that can be found in the LICENSE file.
 +
-+//go:build !goexperiment.systemcrypto
++//go:build !goexperiment.systemcrypto || (goexperiment.darwincrypto && !cgo)
 +
 +package bbig
 +
@@ -1148,7 +1148,7 @@ index 00000000000000..83691d7dd42d51
 +}
 diff --git a/src/crypto/internal/backend/nobackend.go b/src/crypto/internal/backend/nobackend.go
 new file mode 100644
-index 00000000000000..71e0ec9dc25a02
+index 00000000000000..06e19c55345187
 --- /dev/null
 +++ b/src/crypto/internal/backend/nobackend.go
 @@ -0,0 +1,229 @@
@@ -1317,37 +1317,37 @@ index 00000000000000..71e0ec9dc25a02
 +
 +type PublicKeyEd25519 struct{}
 +
-+func (k *PublicKeyEd25519) Bytes() ([]byte, error) {
++func (k PublicKeyEd25519) Bytes() ([]byte, error) {
 +	panic("cryptobackend: not available")
 +}
 +
 +type PrivateKeyEd25519 struct{}
 +
-+func (k *PrivateKeyEd25519) Bytes() ([]byte, error) {
++func (k PrivateKeyEd25519) Bytes() ([]byte, error) {
 +	panic("cryptobackend: not available")
 +}
 +
-+func GenerateKeyEd25519() (*PrivateKeyEd25519, error) {
++func GenerateKeyEd25519() (PrivateKeyEd25519, error) {
 +	panic("cryptobackend: not available")
 +}
 +
-+func NewPrivateKeyEd25119(priv []byte) (*PrivateKeyEd25519, error) {
++func NewPrivateKeyEd25119(priv []byte) (PrivateKeyEd25519, error) {
 +	panic("cryptobackend: not available")
 +}
 +
-+func NewPublicKeyEd25119(pub []byte) (*PublicKeyEd25519, error) {
++func NewPublicKeyEd25119(pub []byte) (PublicKeyEd25519, error) {
 +	panic("cryptobackend: not available")
 +}
 +
-+func NewPrivateKeyEd25519FromSeed(seed []byte) (*PrivateKeyEd25519, error) {
++func NewPrivateKeyEd25519FromSeed(seed []byte) (PrivateKeyEd25519, error) {
 +	panic("cryptobackend: not available")
 +}
 +
-+func SignEd25519(priv *PrivateKeyEd25519, message []byte) ([]byte, error) {
++func SignEd25519(priv PrivateKeyEd25519, message []byte) ([]byte, error) {
 +	panic("cryptobackend: not available")
 +}
 +
-+func VerifyEd25519(pub *PublicKeyEd25519, message, sig []byte) error {
++func VerifyEd25519(pub PublicKeyEd25519, message, sig []byte) error {
 +	panic("cryptobackend: not available")
 +}
 +