crypto/rsa: add c2sp.org/det-keygen test vectors for RSA key generation

Change-Id: I3552a3c9c3de5f2d2d1902682214b1536a6a6964
Reviewed-on: https://go-review.googlesource.com/c/go/+/763020
Auto-Submit: Filippo Valsorda <filippo@golang.org>
Reviewed-by: Michael Pratt <mpratt@google.com>
Reviewed-by: Roland Shoemaker <roland@golang.org>
LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Daniel McCarney <daniel@binaryparadox.net>

src/crypto/internal/fips140/rsa/keygen.go

@@ -14,6 +14,13 @@ import (
 
 // GenerateKey generates a new RSA key pair of the given bit size.
 // bits must be at least 32.
+//
+// It follows the process described at c2sp.org/det-keygen, which is compliant
+// with FIPS 186-5, Appendix A.1, IFC Key Pair Generation and FIPS 186-5,
+// Appendix A.1.3, Generation of Random Primes that are Probably Prime.
+// The prime candidates are drawn from rand, which in production will be the
+// global DRBG, while in tests can be an HMAC_DRBG as specified in
+// c2sp.org/det-keygen, to allow using its tests vectors.
 func GenerateKey(rand io.Reader, bits int) (*PrivateKey, error) {
 	if bits < 32 {
 		return nil, errors.New("rsa: key too small")

src/crypto/rsa/rsa_test.go

@@ -11,13 +11,16 @@ import (
 	"crypto/internal/boring"
 	"crypto/internal/cryptotest"
 	"crypto/internal/fips140"
+	"crypto/internal/fips140/ecdsa"
 	"crypto/rand"
 	. "crypto/rsa"
 	"crypto/sha1"
 	"crypto/sha256"
 	"crypto/sha512"
 	"crypto/x509"
+	"encoding/binary"
 	"encoding/hex"
+	"encoding/json"
 	"encoding/pem"
 	"flag"
 	"fmt"
@@ -131,6 +134,68 @@ func TestTinyKeyGeneration(t *testing.T) {
 	}
 }
 
+func TestKeyGenerationVectors(t *testing.T) {
+	var vectors []struct {
+		Bits  int
+		Seed  []byte
+		PKCS8 []byte `json:"private_key_pkcs8"`
+	}
+	f, err := os.Open("testdata/det-keygen.json")
+	if err != nil {
+		t.Fatalf("failed to open det-keygen.json: %v", err)
+	}
+	defer f.Close()
+	if err := json.NewDecoder(f).Decode(&vectors); err != nil {
+		t.Fatalf("failed to decode keygen.json: %v", err)
+	}
+	for i, v := range vectors {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			t.Setenv("GODEBUG", "cryptocustomrand=1")
+			pers := []byte("det RSA key gen")
+			pers = binary.BigEndian.AppendUint16(pers, uint16(v.Bits))
+			drbg := ecdsa.TestingOnlyNewDRBG(sha256.New, v.Seed, nil, pers)
+			rng := &keyGenTestReader{next: func(p []byte) error {
+				drbg.Generate(p)
+				return nil
+			}}
+			priv, err := GenerateKey(rng, v.Bits)
+			if err != nil {
+				t.Fatalf("GenerateKey: %v", err)
+			}
+			testKeyBasics(t, priv)
+			der, err := x509.MarshalPKCS8PrivateKey(priv)
+			if err != nil {
+				t.Fatalf("MarshalPKCS8PrivateKey: %v", err)
+			}
+			if !bytes.Equal(der, v.PKCS8) {
+				t.Errorf("PKCS8 mismatch:\n%s\nvs\n\n%s", hex.Dump(der), hex.Dump(v.PKCS8))
+			}
+		})
+	}
+}
+
+type keyGenTestReader struct {
+	next func([]byte) error
+}
+
+func (r *keyGenTestReader) Read(p []byte) (n int, err error) {
+	// Neutralize randutil.MaybeReadByte.
+	//
+	// DO NOT COPY this. We *will* break you. We can do this because we're
+	// in the standard library, and can update this along with the
+	// GenerateKey implementation if necessary.
+	//
+	// You have been warned.
+	if len(p) == 1 {
+		return 1, nil
+	}
+
+	if err := r.next(p); err != nil {
+		return 0, err
+	}
+	return len(p), nil
+}
+
 func TestGnuTLSKey(t *testing.T) {
 	t.Setenv("GODEBUG", "rsa1024min=0")
 	// This is a key generated by `certtool --generate-privkey --bits 128`.
@@ -832,7 +897,7 @@ func BenchmarkGenerateKey(b *testing.B) {
 			b.Fatal(err)
 		}
 		for b.Loop() {
-			r := &testPrimeReader{primes: string(primes)}
+			r := benchmarkPrimeReader(string(primes))
 			if _, err := GenerateKey(r, 2048); err != nil {
 				b.Fatal(err)
 			}
@@ -845,7 +910,7 @@ func BenchmarkGenerateKey(b *testing.B) {
 			b.Fatal(err)
 		}
 		for b.Loop() {
-			r := &testPrimeReader{primes: string(primes)}
+			r := benchmarkPrimeReader(string(primes))
 			if _, err := GenerateKey(r, 3072); err != nil {
 				b.Fatal(err)
 			}
@@ -858,7 +923,7 @@ func BenchmarkGenerateKey(b *testing.B) {
 			b.Fatal(err)
 		}
 		for b.Loop() {
-			r := &testPrimeReader{primes: string(primes)}
+			r := benchmarkPrimeReader(string(primes))
 			if _, err := GenerateKey(r, 4096); err != nil {
 				b.Fatal(err)
 			}
@@ -866,41 +931,28 @@ func BenchmarkGenerateKey(b *testing.B) {
 	})
 }
 
-// testPrimeReader feeds prime candidates from a text file,
+// benchmarkPrimeReader feeds prime candidates from a text file,
 // one per line in hex, to GenerateKey.
-type testPrimeReader struct {
-	primes string
-}
-
-func (r *testPrimeReader) Read(p []byte) (n int, err error) {
-	// Neutralize randutil.MaybeReadByte.
-	//
-	// DO NOT COPY this. We *will* break you. We can do this because we're
-	// in the standard library, and can update this along with the
-	// GenerateKey implementation if necessary.
-	//
-	// You have been warned.
-	if len(p) == 1 {
-		return 1, nil
-	}
-
-	var line string
-	for line == "" || line[0] == '#' {
-		var ok bool
-		line, r.primes, ok = strings.Cut(r.primes, "\n")
-		if !ok {
-			return 0, io.EOF
+func benchmarkPrimeReader(primes string) io.Reader {
+	return &keyGenTestReader{next: func(p []byte) error {
+		var line string
+		for line == "" || line[0] == '#' {
+			var ok bool
+			line, primes, ok = strings.Cut(primes, "\n")
+			if !ok {
+				return io.EOF
+			}
 		}
-	}
-	b, err := hex.DecodeString(line)
-	if err != nil {
-		return 0, err
-	}
-	if len(p) != len(b) {
-		return 0, fmt.Errorf("unexpected read length: %d", len(p))
-	}
-	copy(p, b)
-	return len(p), nil
+		b, err := hex.DecodeString(line)
+		if err != nil {
+			return err
+		}
+		if len(p) != len(b) {
+			return fmt.Errorf("unexpected read length: %d", len(p))
+		}
+		copy(p, b)
+		return nil
+	}}
 }
 
 type testEncryptOAEPMessage struct {