crypto: fix fips140=only detection of SHA-3

Both fips140only and the service indicator checks in
crypto/internal/fips140/... expect to type assert to
crypto/internal/fips140/{sha256,sha512,sha3}.Digest.

However, crypto/sha3 returns a wrapper concrete type around sha3.Digest.

Add a new fips140hash.Unwrap function to turn the wrapper into the
underlying sha3.Digest, and use it consistently before calling into
fips140only or the FIPS 140-3 module.

In crypto/rsa, also made the fips140only checks apply consistently after
the Go+BoringCrypto shims, so we can instantiate the hash, and avoid
having to wrap the New function. Note that fips140=only is incompatible
with Go+BoringCrypto.

Fixes #70879

Change-Id: I6a6a4656ec55c3e13f6cbfadb9cf89c0f9183bdc
Reviewed-on: https://go-review.googlesource.com/c/go/+/640855
Auto-Submit: Filippo Valsorda <[email protected]>
Reviewed-by: Roland Shoemaker <[email protected]>
Reviewed-by: Russ Cox <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>

Author: FiloSottile

src/crypto/ecdsa/ecdsa.go

@@ -23,6 +23,7 @@ import (
 	"crypto/internal/boring"
 	"crypto/internal/boring/bbig"
 	"crypto/internal/fips140/ecdsa"
+	"crypto/internal/fips140hash"
 	"crypto/internal/fips140only"
 	"crypto/internal/randutil"
 	"crypto/sha512"
@@ -281,10 +282,11 @@ func signFIPSDeterministic[P ecdsa.Point[P]](c *ecdsa.Curve[P], hashFunc crypto.
 	if err != nil {
 		return nil, err
 	}
-	if fips140only.Enabled && !fips140only.ApprovedHash(hashFunc.New()) {
+	h := fips140hash.UnwrapNew(hashFunc.New)
+	if fips140only.Enabled && !fips140only.ApprovedHash(h()) {
 		return nil, errors.New("crypto/ecdsa: use of hash functions other than SHA-2 or SHA-3 is not allowed in FIPS 140-only mode")
 	}
-	sig, err := ecdsa.SignDeterministic(c, hashFunc.New, k, hash)
+	sig, err := ecdsa.SignDeterministic(c, h, k, hash)
 	if err != nil {
 		return nil, err
 	}

src/crypto/hkdf/hkdf.go

@@ -12,6 +12,7 @@ package hkdf
 
 import (
 	"crypto/internal/fips140/hkdf"
+	"crypto/internal/fips140hash"
 	"crypto/internal/fips140only"
 	"errors"
 	"hash"
@@ -24,10 +25,11 @@ import (
 // Expand invocations and different context values. Most common scenarios,
 // including the generation of multiple keys, should use [Key] instead.
 func Extract[H hash.Hash](h func() H, secret, salt []byte) ([]byte, error) {
-	if err := checkFIPS140Only(h, secret); err != nil {
+	fh := fips140hash.UnwrapNew(h)
+	if err := checkFIPS140Only(fh, secret); err != nil {
 		return nil, err
 	}
-	return hkdf.Extract(h, secret, salt), nil
+	return hkdf.Extract(fh, secret, salt), nil
 }
 
 // Expand derives a key from the given hash, key, and optional context info,
@@ -38,35 +40,37 @@ func Extract[H hash.Hash](h func() H, secret, salt []byte) ([]byte, error) {
 // random or pseudorandom cryptographically strong key. See RFC 5869, Section
 // 3.3. Most common scenarios will want to use [Key] instead.
 func Expand[H hash.Hash](h func() H, pseudorandomKey []byte, info string, keyLength int) ([]byte, error) {
-	if err := checkFIPS140Only(h, pseudorandomKey); err != nil {
+	fh := fips140hash.UnwrapNew(h)
+	if err := checkFIPS140Only(fh, pseudorandomKey); err != nil {
 		return nil, err
 	}
 
-	limit := h().Size() * 255
+	limit := fh().Size() * 255
 	if keyLength > limit {
 		return nil, errors.New("hkdf: requested key length too large")
 	}
 
-	return hkdf.Expand(h, pseudorandomKey, info, keyLength), nil
+	return hkdf.Expand(fh, pseudorandomKey, info, keyLength), nil
 }
 
 // Key derives a key from the given hash, secret, salt and context info,
 // returning a []byte of length keyLength that can be used as cryptographic key.
 // Salt and info can be nil.
 func Key[Hash hash.Hash](h func() Hash, secret, salt []byte, info string, keyLength int) ([]byte, error) {
-	if err := checkFIPS140Only(h, secret); err != nil {
+	fh := fips140hash.UnwrapNew(h)
+	if err := checkFIPS140Only(fh, secret); err != nil {
 		return nil, err
 	}
 
-	limit := h().Size() * 255
+	limit := fh().Size() * 255
 	if keyLength > limit {
 		return nil, errors.New("hkdf: requested key length too large")
 	}
 
-	return hkdf.Key(h, secret, salt, info, keyLength), nil
+	return hkdf.Key(fh, secret, salt, info, keyLength), nil
 }
 
-func checkFIPS140Only[H hash.Hash](h func() H, key []byte) error {
+func checkFIPS140Only[Hash hash.Hash](h func() Hash, key []byte) error {
 	if !fips140only.Enabled {
 		return nil
 	}

src/crypto/hmac/hmac.go

@@ -24,6 +24,7 @@ package hmac
 import (
 	"crypto/internal/boring"
 	"crypto/internal/fips140/hmac"
+	"crypto/internal/fips140hash"
 	"crypto/internal/fips140only"
 	"crypto/subtle"
 	"hash"
@@ -43,6 +44,7 @@ func New(h func() hash.Hash, key []byte) hash.Hash {
 		}
 		// BoringCrypto did not recognize h, so fall through to standard Go code.
 	}
+	h = fips140hash.UnwrapNew(h)
 	if fips140only.Enabled {
 		if len(key) < 112/8 {
 			panic("crypto/hmac: use of keys shorter than 112 bits is not allowed in FIPS 140-only mode")

src/crypto/internal/fips140hash/hash.go

@@ -0,0 +1,34 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fips140hash
+
+import (
+	fsha3 "crypto/internal/fips140/sha3"
+	"crypto/sha3"
+	"hash"
+	_ "unsafe"
+)
+
+//go:linkname sha3Unwrap
+func sha3Unwrap(*sha3.SHA3) *fsha3.Digest
+
+// Unwrap returns h, or a crypto/internal/fips140 inner implementation of h.
+//
+// The return value can be type asserted to one of
+// [crypto/internal/fips140/sha256.Digest],
+// [crypto/internal/fips140/sha512.Digest], or
+// [crypto/internal/fips140/sha3.Digest] if it is a FIPS 140-3 approved hash.
+func Unwrap(h hash.Hash) hash.Hash {
+	if sha3, ok := h.(*sha3.SHA3); ok {
+		return sha3Unwrap(sha3)
+	}
+	return h
+}
+
+// UnwrapNew returns a function that calls newHash and applies [Unwrap] to the
+// return value.
+func UnwrapNew[Hash hash.Hash](newHash func() Hash) func() hash.Hash {
+	return func() hash.Hash { return Unwrap(newHash()) }
+}

src/crypto/pbkdf2/pbkdf2.go

@@ -12,6 +12,7 @@ package pbkdf2
 
 import (
 	"crypto/internal/fips140/pbkdf2"
+	"crypto/internal/fips140hash"
 	"crypto/internal/fips140only"
 	"errors"
 	"hash"
@@ -34,16 +35,17 @@ import (
 // Using a higher iteration count will increase the cost of an exhaustive
 // search but will also make derivation proportionally slower.
 func Key[Hash hash.Hash](h func() Hash, password string, salt []byte, iter, keyLength int) ([]byte, error) {
+	fh := fips140hash.UnwrapNew(h)
 	if fips140only.Enabled {
 		if keyLength < 112/8 {
 			return nil, errors.New("crypto/pbkdf2: use of keys shorter than 112 bits is not allowed in FIPS 140-only mode")
 		}
 		if len(salt) < 128/8 {
 			return nil, errors.New("crypto/pbkdf2: use of salts shorter than 128 bits is not allowed in FIPS 140-only mode")
 		}
-		if !fips140only.ApprovedHash(h()) {
+		if !fips140only.ApprovedHash(fh()) {
 			return nil, errors.New("crypto/pbkdf2: use of hash functions other than SHA-2 or SHA-3 is not allowed in FIPS 140-only mode")
 		}
 	}
-	return pbkdf2.Key(h, password, salt, iter, keyLength)
+	return pbkdf2.Key(fh, password, salt, iter, keyLength)
 }