snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

credentials/tls/certprovider/distributor.go

@@ -0,0 +1,114 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package certprovider
+
+import (
+	"context"
+	"sync"
+
+	"google.golang.org/grpc/internal/grpcsync"
+)
+
+// Distributor makes it easy for provider implementations to furnish new key
+// materials by handling synchronization between the producer and consumers of
+// the key material.
+//
+// Provider implementations which choose to use a Distributor should do the
+// following:
+//   - create a new Distributor using the NewDistributor() function.
+//   - invoke the Set() method whenever they have new key material or errors to
+//     report.
+//   - delegate to the distributor when handing calls to KeyMaterial().
+//   - invoke the Stop() method when they are done using the distributor.
+type Distributor struct {
+	// mu protects the underlying key material.
+	mu   sync.Mutex
+	km   *KeyMaterial
+	pErr error
+
+	// ready channel to unblock KeyMaterial() invocations blocked on
+	// availability of key material.
+	ready *grpcsync.Event
+	// done channel to notify provider implementations and unblock any
+	// KeyMaterial() calls, once the Distributor is closed.
+	closed *grpcsync.Event
+}
+
+// NewDistributor returns a new Distributor.
+func NewDistributor() *Distributor {
+	return &Distributor{
+		ready:  grpcsync.NewEvent(),
+		closed: grpcsync.NewEvent(),
+	}
+}
+
+// Set updates the key material in the distributor with km.
+//
+// Provider implementations which use the distributor must not modify the
+// contents of the KeyMaterial struct pointed to by km.
+//
+// A non-nil err value indicates the error that the provider implementation ran
+// into when trying to fetch key material, and makes it possible to surface the
+// error to the user. A non-nil error value passed here causes distributor's
+// KeyMaterial() method to return nil key material.
+func (d *Distributor) Set(km *KeyMaterial, err error) {
+	d.mu.Lock()
+	d.km = km
+	d.pErr = err
+	if err != nil {
+		// If a non-nil err is passed, we ignore the key material being passed.
+		d.km = nil
+	}
+	d.ready.Fire()
+	d.mu.Unlock()
+}
+
+// KeyMaterial returns the most recent key material provided to the Distributor.
+// If no key material was provided at the time of this call, it will block until
+// the deadline on the context expires or fresh key material arrives.
+func (d *Distributor) KeyMaterial(ctx context.Context) (*KeyMaterial, error) {
+	if d.closed.HasFired() {
+		return nil, errProviderClosed
+	}
+
+	if d.ready.HasFired() {
+		return d.keyMaterial()
+	}
+
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case <-d.closed.Done():
+		return nil, errProviderClosed
+	case <-d.ready.Done():
+		return d.keyMaterial()
+	}
+}
+
+func (d *Distributor) keyMaterial() (*KeyMaterial, error) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	return d.km, d.pErr
+}
+
+// Stop turns down the distributor, releases allocated resources and fails any
+// active KeyMaterial() call waiting for new key material.
+func (d *Distributor) Stop() {
+	d.closed.Fire()
+}

credentials/tls/certprovider/distributor_test.go

@@ -0,0 +1,110 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package certprovider
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+var errProviderTestInternal = errors.New("provider internal error")
+
+// TestDistributorEmpty tries to read key material from an empty distributor and
+// expects the call to timeout.
+func (s) TestDistributorEmpty(t *testing.T) {
+	dist := NewDistributor()
+
+	// This call to KeyMaterial() should timeout because no key material has
+	// been set on the distributor as yet.
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	if err := readAndVerifyKeyMaterial(ctx, dist, nil); !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatal(err)
+	}
+}
+
+// TestDistributor invokes the different methods on the Distributor type and
+// verifies the results.
+func (s) TestDistributor(t *testing.T) {
+	dist := NewDistributor()
+
+	// Read cert/key files from testdata.
+	km1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem")
+
+	// Push key material into the distributor and make sure that a call to
+	// KeyMaterial() returns the expected key material, with both the local
+	// certs and root certs.
+	dist.Set(km1, nil)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := readAndVerifyKeyMaterial(ctx, dist, km1); err != nil {
+		t.Fatal(err)
+	}
+
+	// Push new key material into the distributor and make sure that a call to
+	// KeyMaterial() returns the expected key material, with only root certs.
+	dist.Set(km2, nil)
+	if err := readAndVerifyKeyMaterial(ctx, dist, km2); err != nil {
+		t.Fatal(err)
+	}
+
+	// Push an error into the distributor and make sure that a call to
+	// KeyMaterial() returns that error and nil keyMaterial.
+	dist.Set(km2, errProviderTestInternal)
+	if gotKM, err := dist.KeyMaterial(ctx); gotKM != nil || !errors.Is(err, errProviderTestInternal) {
+		t.Fatalf("KeyMaterial() = {%v, %v}, want {nil, %v}", gotKM, err, errProviderTestInternal)
+	}
+
+	// Stop the distributor and KeyMaterial() should return errProviderClosed.
+	dist.Stop()
+	if km, err := dist.KeyMaterial(ctx); !errors.Is(err, errProviderClosed) {
+		t.Fatalf("KeyMaterial() = {%v, %v}, want {nil, %v}", km, err, errProviderClosed)
+	}
+}
+
+// TestDistributorConcurrency invokes methods on the distributor in parallel. It
+// exercises that the scenario where a distributor's KeyMaterial() method is
+// blocked waiting for keyMaterial, while the Set() method is called from
+// another goroutine. It verifies that the KeyMaterial() method eventually
+// returns with expected keyMaterial.
+func (s) TestDistributorConcurrency(t *testing.T) {
+	dist := NewDistributor()
+
+	// Read cert/key files from testdata.
+	km := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Push key material into the distributor from a goroutine and read from
+	// here to verify that the distributor returns the expected keyMaterial.
+	go func() {
+		// Add a small sleep here to make sure that the call to KeyMaterial()
+		// happens before the call to Set(), thereby the former is blocked till
+		// the latter happens.
+		time.Sleep(100 * time.Microsecond)
+		dist.Set(km, nil)
+	}()
+	if err := readAndVerifyKeyMaterial(ctx, dist, km); err != nil {
+		t.Fatal(err)
+	}
+}

credentials/tls/certprovider/pemfile/builder.go

@@ -0,0 +1,103 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package pemfile
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/types/known/durationpb"
+)
+
+const (
+	// PluginName is the name of the PEM file watcher plugin.
+	PluginName             = "file_watcher"
+	defaultRefreshInterval = 10 * time.Minute
+)
+
+func init() {
+	certprovider.Register(&pluginBuilder{})
+}
+
+type pluginBuilder struct{}
+
+func (p *pluginBuilder) ParseConfig(c any) (*certprovider.BuildableConfig, error) {
+	data, ok := c.(json.RawMessage)
+	if !ok {
+		return nil, fmt.Errorf("meshca: unsupported config type: %T", c)
+	}
+	opts, err := pluginConfigFromJSON(data)
+	if err != nil {
+		return nil, err
+	}
+	return certprovider.NewBuildableConfig(PluginName, opts.canonical(), func(certprovider.BuildOptions) certprovider.Provider {
+		return newProvider(opts)
+	}), nil
+}
+
+func (p *pluginBuilder) Name() string {
+	return PluginName
+}
+
+func pluginConfigFromJSON(jd json.RawMessage) (Options, error) {
+	// The only difference between this anonymous struct and the Options struct
+	// is that the refresh_interval is represented here as a duration proto,
+	// while in the latter a time.Duration is used.
+	cfg := &struct {
+		CertificateFile          string          `json:"certificate_file,omitempty"`
+		PrivateKeyFile           string          `json:"private_key_file,omitempty"`
+		CACertificateFile        string          `json:"ca_certificate_file,omitempty"`
+		SPIFFETrustBundleMapFile string          `json:"spiffe_trust_bundle_map_file,omitempty"`
+		RefreshInterval          json.RawMessage `json:"refresh_interval,omitempty"`
+	}{}
+	if err := json.Unmarshal(jd, cfg); err != nil {
+		return Options{}, fmt.Errorf("pemfile: json.Unmarshal(%s) failed: %v", string(jd), err)
+	}
+	if !envconfig.XDSSPIFFEEnabled {
+		cfg.SPIFFETrustBundleMapFile = ""
+	}
+
+	opts := Options{
+		CertFile:            cfg.CertificateFile,
+		KeyFile:             cfg.PrivateKeyFile,
+		RootFile:            cfg.CACertificateFile,
+		SPIFFEBundleMapFile: cfg.SPIFFETrustBundleMapFile,
+		// Refresh interval is the only field in the configuration for which we
+		// support a default value. We cannot possibly have valid defaults for
+		// file paths to watch. Also, it is valid to specify an empty path for
+		// some of those fields if the user does not want to watch them.
+		RefreshDuration: defaultRefreshInterval,
+	}
+	if cfg.RefreshInterval != nil {
+		dur := &durationpb.Duration{}
+		if err := protojson.Unmarshal(cfg.RefreshInterval, dur); err != nil {
+			return Options{}, fmt.Errorf("pemfile: protojson.Unmarshal(%+v) failed: %v", cfg.RefreshInterval, err)
+		}
+		opts.RefreshDuration = dur.AsDuration()
+	}
+
+	if err := opts.validate(); err != nil {
+		return Options{}, err
+	}
+	return opts, nil
+}

credentials/tls/certprovider/pemfile/builder_test.go

@@ -0,0 +1,162 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package pemfile
+
+import (
+	"encoding/json"
+	"testing"
+
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/testutils"
+)
+
+func TestParseConfig(t *testing.T) {
+	tests := []struct {
+		desc          string
+		input         any
+		wantOutput    string
+		wantErr       bool
+		enabledSpiffe bool
+	}{
+		{
+			desc:    "non JSON input",
+			input:   new(int),
+			wantErr: true,
+		},
+		{
+			desc:    "invalid JSON",
+			input:   json.RawMessage(`bad bad json`),
+			wantErr: true,
+		},
+		{
+			desc:    "JSON input does not match expected",
+			input:   json.RawMessage(`["foo": "bar"]`),
+			wantErr: true,
+		},
+		{
+			desc:    "no credential files",
+			input:   json.RawMessage(`{}`),
+			wantErr: true,
+		},
+		{
+			desc: "only cert file",
+			input: json.RawMessage(`
+			{
+				"certificate_file": "/a/b/cert.pem"
+			}`),
+			wantErr: true,
+		},
+		{
+			desc: "only key file",
+			input: json.RawMessage(`
+			{
+				"private_key_file": "/a/b/key.pem"
+			}`),
+			wantErr: true,
+		},
+		{
+			desc: "cert and key in different directories",
+			input: json.RawMessage(`
+			{
+				"certificate_file": "/b/a/cert.pem",
+				"private_key_file": "/a/b/key.pem"
+			}`),
+			wantErr: true,
+		},
+		{
+			desc: "bad refresh duration",
+			input: json.RawMessage(`
+			{
+				"certificate_file":   "/a/b/cert.pem",
+				"private_key_file":    "/a/b/key.pem",
+				"ca_certificate_file": "/a/b/ca.pem",
+				"refresh_interval":   "duration"
+			}`),
+			wantErr: true,
+		},
+		{
+			desc: "good config with default refresh interval",
+			input: json.RawMessage(`
+			{
+				"certificate_file":   "/a/b/cert.pem",
+				"private_key_file":    "/a/b/key.pem",
+				"ca_certificate_file": "/a/b/ca.pem"
+			}`),
+			wantOutput: "file_watcher:/a/b/cert.pem:/a/b/key.pem:/a/b/ca.pem::10m0s",
+		},
+		{
+			desc: "good config",
+			input: json.RawMessage(`
+			{
+				"certificate_file":   "/a/b/cert.pem",
+				"private_key_file":    "/a/b/key.pem",
+				"ca_certificate_file": "/a/b/ca.pem",
+				"refresh_interval":   "200s"
+			}`),
+			wantOutput: "file_watcher:/a/b/cert.pem:/a/b/key.pem:/a/b/ca.pem::3m20s",
+		},
+		{
+			desc: "good config with spiffe disabled",
+			input: json.RawMessage(`
+			{
+				"certificate_file":   "/a/b/cert.pem",
+				"private_key_file":    "/a/b/key.pem",
+				"ca_certificate_file": "/a/b/ca.pem",
+				"spiffe_trust_bundle_map_file": "/a/b/spiffe_bundle.json",
+				"refresh_interval":   "200s"
+			}`),
+			wantOutput: "file_watcher:/a/b/cert.pem:/a/b/key.pem:/a/b/ca.pem::3m20s",
+		},
+		{
+			desc: "good config with spiffe enabled",
+			input: json.RawMessage(`
+			{
+				"certificate_file":   "/a/b/cert.pem",
+				"private_key_file":    "/a/b/key.pem",
+				"ca_certificate_file": "/a/b/ca.pem",
+				"spiffe_trust_bundle_map_file": "/a/b/spiffe_bundle.json",
+				"refresh_interval":   "200s"
+			}`),
+			wantOutput:    "file_watcher:/a/b/cert.pem:/a/b/key.pem:/a/b/ca.pem:/a/b/spiffe_bundle.json:3m20s",
+			enabledSpiffe: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			if test.enabledSpiffe {
+				testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, true)
+			}
+			builder := &pluginBuilder{}
+
+			bc, err := builder.ParseConfig(test.input)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("ParseConfig(%+v) failed: %v", test.input, err)
+			}
+			if test.wantErr {
+				return
+			}
+
+			gotConfig := bc.String()
+			if gotConfig != test.wantOutput {
+				t.Fatalf("ParseConfig(%v) = %s, want %s", test.input, gotConfig, test.wantOutput)
+			}
+		})
+	}
+}

credentials/tls/certprovider/pemfile/watcher.go

@@ -0,0 +1,297 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package pemfile provides a file watching certificate provider plugin
+// implementation which works for files with PEM contents.
+//
+// # Experimental
+//
+// Notice: All APIs in this package are experimental and may be removed in a
+// later release.
+package pemfile
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/credentials/spiffe"
+)
+
+const defaultCertRefreshDuration = 1 * time.Hour
+
+var (
+	// For overriding from unit tests.
+	newDistributor = func() distributor { return certprovider.NewDistributor() }
+
+	logger = grpclog.Component("pemfile")
+)
+
+// Options configures a certificate provider plugin that watches a specified set
+// of files that contain certificates and keys in PEM format.
+type Options struct {
+	// CertFile is the file that holds the identity certificate.
+	// Optional. If this is set, KeyFile must also be set.
+	CertFile string
+	// KeyFile is the file that holds identity private key.
+	// Optional. If this is set, CertFile must also be set.
+	KeyFile string
+	// RootFile is the file that holds trusted root certificate(s).
+	// Optional.
+	RootFile string
+	// SPIFFEBundleMapFile is the file that holds the spiffe bundle map.
+	// If a given provider configures both the RootFile and the
+	// SPIFFEBundleMapFile, the SPIFFEBundleMapFile will be preferred.
+	// Optional.
+	SPIFFEBundleMapFile string
+	// RefreshDuration is the amount of time the plugin waits before checking
+	// for updates in the specified files.
+	// Optional. If not set, a default value (1 hour) will be used.
+	RefreshDuration time.Duration
+}
+
+func (o Options) canonical() []byte {
+	return []byte(fmt.Sprintf("%s:%s:%s:%s:%s", o.CertFile, o.KeyFile, o.RootFile, o.SPIFFEBundleMapFile, o.RefreshDuration))
+}
+
+func (o Options) validate() error {
+	if o.CertFile == "" && o.KeyFile == "" && o.RootFile == "" && o.SPIFFEBundleMapFile == "" {
+		return fmt.Errorf("pemfile: at least one credential file needs to be specified")
+	}
+	if keySpecified, certSpecified := o.KeyFile != "", o.CertFile != ""; keySpecified != certSpecified {
+		return fmt.Errorf("pemfile: private key file and identity cert file should be both specified or not specified")
+	}
+	// C-core has a limitation that they cannot verify that a certificate file
+	// matches a key file. So, the only way to get around this is to make sure
+	// that both files are in the same directory and that they do an atomic
+	// read. Even though Java/Go do not have this limitation, we want the
+	// overall plugin behavior to be consistent across languages.
+	if certDir, keyDir := filepath.Dir(o.CertFile), filepath.Dir(o.KeyFile); certDir != keyDir {
+		return errors.New("pemfile: certificate and key file must be in the same directory")
+	}
+	return nil
+}
+
+// NewProvider returns a new certificate provider plugin that is configured to
+// watch the PEM files specified in the passed in options.
+func NewProvider(o Options) (certprovider.Provider, error) {
+	if err := o.validate(); err != nil {
+		return nil, err
+	}
+	return newProvider(o), nil
+}
+
+// newProvider is used to create a new certificate provider plugin after
+// validating the options, and hence does not return an error.
+func newProvider(o Options) certprovider.Provider {
+	if o.RefreshDuration == 0 {
+		o.RefreshDuration = defaultCertRefreshDuration
+	}
+
+	provider := &watcher{opts: o}
+	if o.CertFile != "" && o.KeyFile != "" {
+		provider.identityDistributor = newDistributor()
+	}
+	if o.RootFile != "" || o.SPIFFEBundleMapFile != "" {
+		provider.rootDistributor = newDistributor()
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	provider.cancel = cancel
+	go provider.run(ctx)
+	return provider
+}
+
+// watcher is a certificate provider plugin that implements the
+// certprovider.Provider interface. It watches a set of certificate and key
+// files and provides the most up-to-date key material for consumption by
+// credentials implementation.
+type watcher struct {
+	identityDistributor         distributor
+	rootDistributor             distributor
+	opts                        Options
+	certFileContents            []byte
+	keyFileContents             []byte
+	rootFileContents            []byte
+	spiffeBundleMapFileContents []byte
+	cancel                      context.CancelFunc
+}
+
+// distributor wraps the methods on certprovider.Distributor which are used by
+// the plugin. This is very useful in tests which need to know exactly when the
+// plugin updates its key material.
+type distributor interface {
+	KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error)
+	Set(km *certprovider.KeyMaterial, err error)
+	Stop()
+}
+
+// updateIdentityDistributor checks if the cert/key files that the plugin is
+// watching have changed, and if so, reads the new contents and updates the
+// identityDistributor with the new key material.
+//
+// Skips updates when file reading or parsing fails.
+// TODO(easwars): Retry with limit (on the number of retries or the amount of
+// time) upon failures.
+func (w *watcher) updateIdentityDistributor() {
+	if w.identityDistributor == nil {
+		return
+	}
+
+	certFileContents, err := os.ReadFile(w.opts.CertFile)
+	if err != nil {
+		logger.Warningf("certFile (%s) read failed: %v", w.opts.CertFile, err)
+		return
+	}
+	keyFileContents, err := os.ReadFile(w.opts.KeyFile)
+	if err != nil {
+		logger.Warningf("keyFile (%s) read failed: %v", w.opts.KeyFile, err)
+		return
+	}
+	// If the file contents have not changed, skip updating the distributor.
+	if bytes.Equal(w.certFileContents, certFileContents) && bytes.Equal(w.keyFileContents, keyFileContents) {
+		return
+	}
+
+	cert, err := tls.X509KeyPair(certFileContents, keyFileContents)
+	if err != nil {
+		logger.Warningf("tls.X509KeyPair(%q, %q) failed: %v", certFileContents, keyFileContents, err)
+		return
+	}
+	w.certFileContents = certFileContents
+	w.keyFileContents = keyFileContents
+	w.identityDistributor.Set(&certprovider.KeyMaterial{Certs: []tls.Certificate{cert}}, nil)
+}
+
+// updateRootDistributor checks if the root cert file that the plugin is
+// watching hs changed, and if so, updates the rootDistributor with the new key
+// material.
+//
+// Skips updates when root cert reading or parsing fails.
+// TODO(easwars): Retry with limit (on the number of retries or the amount of
+// time) upon failures.
+func (w *watcher) updateRootDistributor() {
+	if w.rootDistributor == nil {
+		return
+	}
+
+	// If SPIFFEBundleMap is set, use it and DON'T use the RootFile, even if it
+	// fails
+	if w.opts.SPIFFEBundleMapFile != "" {
+		w.maybeUpdateSPIFFEBundleMap()
+	} else {
+		w.maybeUpdateRootFile()
+	}
+}
+
+func (w *watcher) maybeUpdateSPIFFEBundleMap() {
+	spiffeBundleMapContents, err := os.ReadFile(w.opts.SPIFFEBundleMapFile)
+	if err != nil {
+		logger.Warningf("spiffeBundleMapFile (%s) read failed: %v", w.opts.SPIFFEBundleMapFile, err)
+		return
+	}
+	// If the file contents have not changed, skip updating the distributor.
+	if bytes.Equal(w.spiffeBundleMapFileContents, spiffeBundleMapContents) {
+		return
+	}
+	bundleMap, err := spiffe.BundleMapFromBytes(spiffeBundleMapContents)
+	if err != nil {
+		logger.Warning("Failed to parse spiffe bundle map")
+		return
+	}
+	w.spiffeBundleMapFileContents = spiffeBundleMapContents
+	w.rootDistributor.Set(&certprovider.KeyMaterial{SPIFFEBundleMap: bundleMap}, nil)
+}
+
+func (w *watcher) maybeUpdateRootFile() {
+	rootFileContents, err := os.ReadFile(w.opts.RootFile)
+	if err != nil {
+		logger.Warningf("rootFile (%s) read failed: %v", w.opts.RootFile, err)
+		return
+	}
+	trustPool := x509.NewCertPool()
+	if !trustPool.AppendCertsFromPEM(rootFileContents) {
+		logger.Warning("Failed to parse root certificate")
+		return
+	}
+	// If the file contents have not changed, skip updating the distributor.
+	if bytes.Equal(w.rootFileContents, rootFileContents) {
+		return
+	}
+
+	w.rootFileContents = rootFileContents
+	w.rootDistributor.Set(&certprovider.KeyMaterial{Roots: trustPool}, nil)
+}
+
+// run is a long running goroutine which watches the configured files for
+// changes, and pushes new key material into the appropriate distributors which
+// is returned from calls to KeyMaterial().
+func (w *watcher) run(ctx context.Context) {
+	ticker := time.NewTicker(w.opts.RefreshDuration)
+	for {
+		w.updateIdentityDistributor()
+		w.updateRootDistributor()
+		select {
+		case <-ctx.Done():
+			ticker.Stop()
+			if w.identityDistributor != nil {
+				w.identityDistributor.Stop()
+			}
+			if w.rootDistributor != nil {
+				w.rootDistributor.Stop()
+			}
+			return
+		case <-ticker.C:
+		}
+	}
+}
+
+// KeyMaterial returns the key material sourced by the watcher.
+// Callers are expected to use the returned value as read-only.
+func (w *watcher) KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) {
+	km := &certprovider.KeyMaterial{}
+	if w.identityDistributor != nil {
+		identityKM, err := w.identityDistributor.KeyMaterial(ctx)
+		if err != nil {
+			return nil, err
+		}
+		km.Certs = identityKM.Certs
+	}
+	if w.rootDistributor != nil {
+		rootKM, err := w.rootDistributor.KeyMaterial(ctx)
+		if err != nil {
+			return nil, err
+		}
+		km.SPIFFEBundleMap = rootKM.SPIFFEBundleMap
+		km.Roots = rootKM.Roots
+	}
+	return km, nil
+}
+
+// Close cleans up resources allocated by the watcher.
+func (w *watcher) Close() {
+	w.cancel()
+}

credentials/tls/certprovider/pemfile/watcher_test.go

@@ -0,0 +1,629 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package pemfile
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/testdata"
+)
+
+const (
+	// These are the names of files inside temporary directories, which the
+	// plugin is asked to watch.
+	certFile         = "cert.pem"
+	keyFile          = "key.pem"
+	rootFile         = "ca.pem"
+	spiffeBundleFile = "spiffebundle.json"
+
+	defaultTestRefreshDuration = 100 * time.Millisecond
+	defaultTestTimeout         = 5 * time.Second
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func compareKeyMaterial(got, want *certprovider.KeyMaterial) error {
+	if len(got.Certs) != len(want.Certs) {
+		return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want)
+	}
+	for i := 0; i < len(got.Certs); i++ {
+		if !got.Certs[i].Leaf.Equal(want.Certs[i].Leaf) {
+			return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want)
+		}
+	}
+
+	if gotR, wantR := got.Roots, want.Roots; !gotR.Equal(wantR) {
+		return fmt.Errorf("keyMaterial roots = %v, want %v", gotR, wantR)
+	}
+
+	if gotBundle, wantBundle := got.SPIFFEBundleMap, want.SPIFFEBundleMap; !cmp.Equal(gotBundle, wantBundle) {
+		return fmt.Errorf("keyMaterial spiffe bundle map = %v, want %v", gotBundle, wantBundle)
+	}
+
+	return nil
+}
+
+// TestNewProvider tests the NewProvider() function with different inputs.
+func (s) TestNewProvider(t *testing.T) {
+	tests := []struct {
+		desc      string
+		options   Options
+		wantError bool
+	}{
+		{
+			desc:      "No credential files specified",
+			options:   Options{},
+			wantError: true,
+		},
+		{
+			desc: "Only identity cert is specified",
+			options: Options{
+				CertFile: testdata.Path("x509/client1_cert.pem"),
+			},
+			wantError: true,
+		},
+		{
+			desc: "Only identity key is specified",
+			options: Options{
+				KeyFile: testdata.Path("x509/client1_key.pem"),
+			},
+			wantError: true,
+		},
+		{
+			desc: "Identity cert/key pair is specified",
+			options: Options{
+				KeyFile:  testdata.Path("x509/client1_key.pem"),
+				CertFile: testdata.Path("x509/client1_cert.pem"),
+			},
+		},
+		{
+			desc: "Only root certs are specified",
+			options: Options{
+				RootFile: testdata.Path("x509/client_ca_cert.pem"),
+			},
+		},
+		{
+			desc: "Only spiffe bundle map specified",
+			options: Options{
+				SPIFFEBundleMapFile: testdata.Path("spiffe/spiffebundle.json"),
+			},
+		},
+		{
+			desc: "Everything is specified",
+			options: Options{
+				KeyFile:             testdata.Path("x509/client1_key.pem"),
+				CertFile:            testdata.Path("x509/client1_cert.pem"),
+				RootFile:            testdata.Path("x509/client_ca_cert.pem"),
+				SPIFFEBundleMapFile: testdata.Path("spiffe/spiffebundle.json"),
+			},
+			wantError: false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			provider, err := NewProvider(test.options)
+			if (err != nil) != test.wantError {
+				t.Fatalf("NewProvider(%v) = %v, want %v", test.options, err, test.wantError)
+			}
+			if err != nil {
+				return
+			}
+			provider.Close()
+		})
+	}
+}
+
+// wrappedDistributor wraps a distributor and pushes on a channel whenever new
+// key material is pushed to the distributor.
+type wrappedDistributor struct {
+	*certprovider.Distributor
+	distCh *testutils.Channel
+}
+
+func newWrappedDistributor(distCh *testutils.Channel) *wrappedDistributor {
+	return &wrappedDistributor{
+		distCh:      distCh,
+		Distributor: certprovider.NewDistributor(),
+	}
+}
+
+func (wd *wrappedDistributor) Set(km *certprovider.KeyMaterial, err error) {
+	wd.Distributor.Set(km, err)
+	wd.distCh.Send(nil)
+}
+
+func createTmpFile(t *testing.T, src, dst string) {
+	t.Helper()
+
+	data, err := os.ReadFile(src)
+	if err != nil {
+		t.Fatalf("os.ReadFile(%q) failed: %v", src, err)
+	}
+	if err := os.WriteFile(dst, data, os.ModePerm); err != nil {
+		t.Fatalf("os.WriteFile(%q) failed: %v", dst, err)
+	}
+	t.Logf("Wrote file at: %s", dst)
+	t.Logf("%s", string(data))
+}
+
+func removeTmpFile(t *testing.T, filePath string) {
+	t.Helper()
+	if err := os.Remove(filePath); err != nil {
+		t.Fatalf("os.RemoveFIle(%q) failed: %v", filePath, err)
+	}
+	t.Logf("Removed file at: %s", filePath)
+}
+
+// createTempDirWithFiles creates a temporary directory under the system default
+// tempDir with the given dirSuffix. It also reads from certSrc, keySrc and
+// rootSrc files are creates appropriate files under the newly create tempDir.
+// Returns the name of the created tempDir.
+func createTmpDirWithFiles(t *testing.T, dirSuffix, certSrc, keySrc, rootSrc, spiffeBundleSrc string) string {
+	t.Helper()
+
+	// Create a temp directory. Passing an empty string for the first argument
+	// uses the system temp directory.
+	dir, err := os.MkdirTemp("", dirSuffix)
+	if err != nil {
+		t.Fatalf("os.MkdirTemp() failed: %v", err)
+	}
+	t.Logf("Using tmpdir: %s", dir)
+
+	createTmpFile(t, testdata.Path(certSrc), path.Join(dir, certFile))
+	createTmpFile(t, testdata.Path(keySrc), path.Join(dir, keyFile))
+	createTmpFile(t, testdata.Path(rootSrc), path.Join(dir, rootFile))
+	createTmpFile(t, testdata.Path(spiffeBundleSrc), path.Join(dir, spiffeBundleFile))
+	return dir
+}
+
+// initializeProvider performs setup steps common to all tests (except the one
+// which uses symlinks).
+func initializeProvider(t *testing.T, testName string, useSPIFFEBundle bool) (string, certprovider.Provider, *testutils.Channel, func()) {
+	t.Helper()
+
+	// Override the newDistributor to one which pushes on a channel that we
+	// can block on.
+	origDistributorFunc := newDistributor
+	distCh := testutils.NewChannel()
+	d := newWrappedDistributor(distCh)
+	newDistributor = func() distributor { return d }
+
+	// Create a new provider to watch the files in tmpdir.
+	dir := createTmpDirWithFiles(t, testName+"*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/client_ca_cert.pem", "spiffe/spiffebundle.json")
+	opts := Options{
+		CertFile:        path.Join(dir, certFile),
+		KeyFile:         path.Join(dir, keyFile),
+		RootFile:        path.Join(dir, rootFile),
+		RefreshDuration: defaultTestRefreshDuration,
+	}
+	if useSPIFFEBundle {
+		opts.SPIFFEBundleMapFile = path.Join(dir, spiffeBundleFile)
+	}
+	prov, err := NewProvider(opts)
+	if err != nil {
+		t.Fatalf("NewProvider(%+v) failed: %v", opts, err)
+	}
+
+	// Make sure the provider picks up the files and pushes the key material on
+	// to the distributors.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	for i := 0; i < 2; i++ {
+		// Since we have root and identity certs, we need to make sure the
+		// update is pushed on both of them.
+		if _, err := distCh.Receive(ctx); err != nil {
+			t.Fatalf("Timeout waiting for provider to read files and push key material to distributor: %v", err)
+		}
+	}
+
+	return dir, prov, distCh, func() {
+		newDistributor = origDistributorFunc
+		prov.Close()
+	}
+}
+
+// TestProvider_NoUpdate tests the case where a file watcher plugin is created
+// successfully, and the underlying files do not change. Verifies that the
+// plugin does not push new updates to the distributor in this case.
+func (s) TestProvider_NoUpdate(t *testing.T) {
+	baseName := "no_update"
+	for _, useSPIFFEBundle := range []bool{true, false} {
+		testName := baseName
+		if useSPIFFEBundle {
+			testName = testName + "_" + "withSPIFFEBundle"
+		}
+		t.Run(testName, func(t *testing.T) {
+			_, prov, distCh, cancel := initializeProvider(t, "no_update", useSPIFFEBundle)
+			defer cancel()
+
+			// Make sure the provider is healthy and returns key material.
+			ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cc()
+			if _, err := prov.KeyMaterial(ctx); err != nil {
+				t.Fatalf("provider.KeyMaterial() failed: %v", err)
+			}
+
+			// Files haven't change. Make sure no updates are pushed by the provider.
+			sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration)
+			defer sc()
+			if _, err := distCh.Receive(sCtx); err == nil {
+				t.Fatal("New key material pushed to distributor when underlying files did not change")
+			}
+		})
+	}
+}
+
+// TestProvider_UpdateSuccess tests the case where a file watcher plugin is
+// created successfully and the underlying files change. Verifies that the
+// changes are picked up by the provider.
+func (s) TestProvider_UpdateSuccess(t *testing.T) {
+	baseName := "update_success"
+	for _, useSPIFFEBundle := range []bool{true, false} {
+		testName := baseName
+		if useSPIFFEBundle {
+			testName = testName + "_" + "withSPIFFEBundle"
+		}
+		t.Run(testName, func(t *testing.T) {
+			dir, prov, distCh, cancel := initializeProvider(t, "update_success", useSPIFFEBundle)
+			defer cancel()
+
+			// Make sure the provider is healthy and returns key material.
+			ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cc()
+			km1, err := prov.KeyMaterial(ctx)
+			if err != nil {
+				t.Fatalf("provider.KeyMaterial() failed: %v", err)
+			}
+
+			// Change only the root file.
+			if useSPIFFEBundle {
+				createTmpFile(t, testdata.Path("spiffe/spiffebundle2.json"), path.Join(dir, spiffeBundleFile))
+			} else {
+				createTmpFile(t, testdata.Path("x509/server_ca_cert.pem"), path.Join(dir, rootFile))
+			}
+			if _, err := distCh.Receive(ctx); err != nil {
+				t.Fatal("Timeout waiting for new key material to be pushed to the distributor")
+			}
+
+			// Make sure update is picked up.
+			km2, err := prov.KeyMaterial(ctx)
+			if err != nil {
+				t.Fatalf("provider.KeyMaterial() failed: %v", err)
+			}
+			if err := compareKeyMaterial(km1, km2); err == nil {
+				t.Fatal("Expected provider to return new key material after update to underlying file")
+			}
+
+			// Change only cert/key files.
+			createTmpFile(t, testdata.Path("x509/client2_cert.pem"), path.Join(dir, certFile))
+			createTmpFile(t, testdata.Path("x509/client2_key.pem"), path.Join(dir, keyFile))
+			if _, err := distCh.Receive(ctx); err != nil {
+				t.Fatal("Timeout waiting for new key material to be pushed to the distributor")
+			}
+
+			// Make sure update is picked up.
+			km3, err := prov.KeyMaterial(ctx)
+			if err != nil {
+				t.Fatalf("provider.KeyMaterial() failed: %v", err)
+			}
+			if err := compareKeyMaterial(km2, km3); err == nil {
+				t.Fatal("Expected provider to return new key material after update to underlying file")
+			}
+		})
+	}
+}
+
+// TestProvider_UpdateSuccessWithSymlink tests the case where a file watcher
+// plugin is created successfully to watch files through a symlink and the
+// symlink is updates to point to new files. Verifies that the changes are
+// picked up by the provider.
+func (s) TestProvider_UpdateSuccessWithSymlink(t *testing.T) {
+	baseName := "update_with_symlink"
+	for _, useSPIFFEBundle := range []bool{true, false} {
+		testName := baseName
+		if useSPIFFEBundle {
+			testName = testName + "_" + "withSPIFFEBundle"
+		}
+		t.Run(testName, func(t *testing.T) {
+			// Override the newDistributor to one which pushes on a channel that we
+			// can block on.
+			origDistributorFunc := newDistributor
+			distCh := testutils.NewChannel()
+			d := newWrappedDistributor(distCh)
+			newDistributor = func() distributor { return d }
+			defer func() { newDistributor = origDistributorFunc }()
+
+			// Create two tempDirs with different files.
+			dir1 := createTmpDirWithFiles(t, "update_with_symlink1_*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/client_ca_cert.pem", "spiffe/spiffebundle.json")
+			dir2 := createTmpDirWithFiles(t, "update_with_symlink2_*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/server_ca_cert.pem", "spiffe/spiffebundle2.json")
+
+			// Create a symlink under a new tempdir, and make it point to dir1.
+			tmpdir, err := os.MkdirTemp("", "test_symlink_*")
+			if err != nil {
+				t.Fatalf("os.MkdirTemp() failed: %v", err)
+			}
+			symLinkName := path.Join(tmpdir, "test_symlink")
+			if err := os.Symlink(dir1, symLinkName); err != nil {
+				t.Fatalf("Failed to create symlink to %q: %v", dir1, err)
+			}
+
+			// Create a provider which watches the files pointed to by the symlink.
+			opts := Options{
+				CertFile:            path.Join(symLinkName, certFile),
+				KeyFile:             path.Join(symLinkName, keyFile),
+				RootFile:            path.Join(symLinkName, rootFile),
+				SPIFFEBundleMapFile: path.Join(symLinkName, spiffeBundleFile),
+				RefreshDuration:     defaultTestRefreshDuration,
+			}
+			if useSPIFFEBundle {
+				opts.SPIFFEBundleMapFile = path.Join(symLinkName, spiffeBundleFile)
+			}
+			prov, err := NewProvider(opts)
+			if err != nil {
+				t.Fatalf("NewProvider(%+v) failed: %v", opts, err)
+			}
+			defer prov.Close()
+
+			// Make sure the provider picks up the files and pushes the key material on
+			// to the distributors.
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			for i := 0; i < 2; i++ {
+				// Since we have root and identity certs, we need to make sure the
+				// update is pushed on both of them.
+				if _, err := distCh.Receive(ctx); err != nil {
+					t.Fatalf("Timeout waiting for provider to read files and push key material to distributor: %v", err)
+				}
+			}
+			km1, err := prov.KeyMaterial(ctx)
+			if err != nil {
+				t.Fatalf("provider.KeyMaterial() failed: %v", err)
+			}
+
+			// Update the symlink to point to dir2.
+			symLinkTmpName := path.Join(tmpdir, "test_symlink.tmp")
+			if err := os.Symlink(dir2, symLinkTmpName); err != nil {
+				t.Fatalf("Failed to create symlink to %q: %v", dir2, err)
+			}
+			if err := os.Rename(symLinkTmpName, symLinkName); err != nil {
+				t.Fatalf("Failed to update symlink: %v", err)
+			}
+
+			// Make sure the provider picks up the new files and pushes the key material
+			// on to the distributors.
+			for i := 0; i < 2; i++ {
+				// Since we have root and identity certs, we need to make sure the
+				// update is pushed on both of them.
+				if _, err := distCh.Receive(ctx); err != nil {
+					t.Fatalf("Timeout waiting for provider to read files and push key material to distributor: %v", err)
+				}
+			}
+			km2, err := prov.KeyMaterial(ctx)
+			if err != nil {
+				t.Fatalf("provider.KeyMaterial() failed: %v", err)
+			}
+
+			if err := compareKeyMaterial(km1, km2); err == nil {
+				t.Fatal("Expected provider to return new key material after symlink update")
+			}
+		})
+	}
+}
+
+// TestProvider_UpdateFailure_ThenSuccess tests the case where updating cert/key
+// files fail. Verifies that the failed update does not push anything on the
+// distributor. Then the update succeeds, and the test verifies that the key
+// material is updated.
+func (s) TestProvider_UpdateFailure_ThenSuccess(t *testing.T) {
+	dir, prov, distCh, cancel := initializeProvider(t, "update_failure", false)
+	defer cancel()
+
+	// Make sure the provider is healthy and returns key material.
+	ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cc()
+	km1, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+
+	// Update only the cert file. The key file is left unchanged. This should
+	// lead to these two files being not compatible with each other. This
+	// simulates the case where the watching goroutine might catch the files in
+	// the midst of an update.
+	createTmpFile(t, testdata.Path("x509/server1_cert.pem"), path.Join(dir, certFile))
+
+	// Since the last update left the files in an incompatible state, the update
+	// should not be picked up by our provider.
+	sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration)
+	defer sc()
+	if _, err := distCh.Receive(sCtx); err == nil {
+		t.Fatal("New key material pushed to distributor when underlying files did not change")
+	}
+
+	// The provider should return key material corresponding to the old state.
+	km2, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+	if err := compareKeyMaterial(km1, km2); err != nil {
+		t.Fatalf("Expected provider to not update key material: %v", err)
+	}
+
+	// Update the key file to match the cert file.
+	createTmpFile(t, testdata.Path("x509/server1_key.pem"), path.Join(dir, keyFile))
+
+	// Make sure update is picked up.
+	if _, err := distCh.Receive(ctx); err != nil {
+		t.Fatal("Timeout waiting for new key material to be pushed to the distributor")
+	}
+	km3, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+	if err := compareKeyMaterial(km2, km3); err == nil {
+		t.Fatal("Expected provider to return new key material after update to underlying file")
+	}
+}
+
+// TestProvider_UpdateFailure_ThenSuccess tests the case where updating cert/key
+// files fail. Verifies that the failed update does not push anything on the
+// distributor. Then the update succeeds, and the test verifies that the key
+// material is updated.
+func (s) TestProvider_UpdateFailureSPIFFE(t *testing.T) {
+	tests := []struct {
+		name    string
+		badFile string
+	}{
+		{
+			name:    "malformed spiffe",
+			badFile: "spiffe/spiffebundle_malformed.json",
+		},
+		{
+			name:    "invalid bundle",
+			badFile: "spiffe/spiffebundle_wrong_kty.json",
+		},
+		{
+			name:    "cert in the x5c field is invalid",
+			badFile: "spiffe/spiffebundle_corrupted_cert.json",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			dir, prov, distCh, cancel := initializeProvider(t, tc.name, true)
+			defer cancel()
+
+			// Make sure the provider is healthy and returns key material.
+			ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cc()
+			km1, err := prov.KeyMaterial(ctx)
+			if err != nil {
+				t.Fatalf("provider.KeyMaterial() failed: %v", err)
+			}
+
+			// Update the file with a bad update
+			createTmpFile(t, testdata.Path(tc.badFile), path.Join(dir, spiffeBundleFile))
+
+			// Since the last update left the files in an incompatible state, the update
+			// should not be picked up by our provider.
+			sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration)
+			defer sc()
+			if _, err := distCh.Receive(sCtx); err == nil {
+				t.Fatal("New key material pushed to distributor when underlying files did not change")
+			}
+
+			// The provider should return key material corresponding to the old state.
+			km2, err := prov.KeyMaterial(ctx)
+			if err != nil {
+				t.Fatalf("provider.KeyMaterial() failed: %v", err)
+			}
+			if err := compareKeyMaterial(km1, km2); err != nil {
+				t.Fatalf("Expected provider to not update key material: %v", err)
+			}
+		})
+	}
+}
+
+// TestProvider_UpdateFailure_ThenSuccess tests the case where updating cert/key
+// files fail. Verifies that the failed update does not push anything on the
+// distributor. Then the update succeeds, and the test verifies that the key
+// material is updated.
+func (s) TestProvider_UpdateFailureSPIFFE_MissingFile(t *testing.T) {
+	dir, prov, distCh, cancel := initializeProvider(t, "Delete spiffe file being read", true)
+	defer cancel()
+
+	// Make sure the provider is healthy and returns key material.
+	ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cc()
+	km1, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+
+	// Remove the file that we are reading
+	removeTmpFile(t, path.Join(dir, spiffeBundleFile))
+
+	// Since the last update left the files in an incompatible state, the update
+	// should not be picked up by our provider.
+	sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration)
+	defer sc()
+	if _, err := distCh.Receive(sCtx); err == nil {
+		t.Fatal("new key material pushed to distributor when underlying files did not change")
+	}
+
+	// The provider should return key material corresponding to the old state.
+	km2, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+	if err := compareKeyMaterial(km1, km2); err != nil {
+		t.Fatalf("expected provider to not update key material: %v", err)
+	}
+}
+
+// TestProvider_UpdateFailure_ThenSuccess tests the case where updating cert/key
+// files fail. Verifies that the failed update does not push anything on the
+// distributor. Then the update succeeds, and the test verifies that the key
+// material is updated.
+func (s) TestProvider_UpdateFailureRoot_MissingFile(t *testing.T) {
+	dir, prov, distCh, cancel := initializeProvider(t, "Delete root file being read", false)
+	defer cancel()
+
+	// Make sure the provider is healthy and returns key material.
+	ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cc()
+	km1, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+
+	// Remove the file that we are reading
+	removeTmpFile(t, path.Join(dir, rootFile))
+
+	// Since the last update left the files in an incompatible state, the update
+	// should not be picked up by our provider.
+	sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration)
+	defer sc()
+	if _, err := distCh.Receive(sCtx); err == nil {
+		t.Fatal("new key material pushed to distributor when underlying files did not change")
+	}
+
+	// The provider should return key material corresponding to the old state.
+	km2, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() failed: %v", err)
+	}
+	if err := compareKeyMaterial(km1, km2); err != nil {
+		t.Fatalf("expected provider to not update key material: %v", err)
+	}
+}

credentials/tls/certprovider/provider.go

@@ -0,0 +1,115 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package certprovider defines APIs for Certificate Providers in gRPC.
+//
+// # Experimental
+//
+// Notice: All APIs in this package are experimental and may be removed in a
+// later release.
+package certprovider
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+
+	"github.com/spiffe/go-spiffe/v2/bundle/spiffebundle"
+	"google.golang.org/grpc/internal"
+)
+
+func init() {
+	internal.GetCertificateProviderBuilder = getBuilder
+}
+
+var (
+	// errProviderClosed is returned by Distributor.KeyMaterial when it is
+	// closed.
+	errProviderClosed = errors.New("provider instance is closed")
+
+	// m is a map from name to Provider builder.
+	m = make(map[string]Builder)
+)
+
+// Register registers the Provider builder, whose name as returned by its Name()
+// method will be used as the name registered with this builder. Registered
+// Builders are used by the Store to create Providers.
+func Register(b Builder) {
+	m[b.Name()] = b
+}
+
+// getBuilder returns the Provider builder registered with the given name.
+// If no builder is registered with the provided name, nil will be returned.
+func getBuilder(name string) Builder {
+	if b, ok := m[name]; ok {
+		return b
+	}
+	return nil
+}
+
+// Builder creates a Provider.
+type Builder interface {
+	// ParseConfig parses the given config, which is in a format specific to individual
+	// implementations, and returns a BuildableConfig on success.
+	ParseConfig(any) (*BuildableConfig, error)
+
+	// Name returns the name of providers built by this builder.
+	Name() string
+}
+
+// Provider makes it possible to keep channel credential implementations up to
+// date with secrets that they rely on to secure communications on the
+// underlying channel.
+//
+// Provider implementations are free to rely on local or remote sources to fetch
+// the latest secrets, and free to share any state between different
+// instantiations as they deem fit.
+type Provider interface {
+	// KeyMaterial returns the key material sourced by the Provider.
+	// Callers are expected to use the returned value as read-only.
+	KeyMaterial(ctx context.Context) (*KeyMaterial, error)
+
+	// Close cleans up resources allocated by the Provider.
+	Close()
+}
+
+// KeyMaterial wraps the certificates and keys returned by a Provider instance.
+type KeyMaterial struct {
+	// Certs contains a slice of cert/key pairs used to prove local identity.
+	Certs []tls.Certificate
+	// Roots contains the set of trusted roots to validate the peer's identity.
+	// This field will only be used if the `SPIFFEBundleMap` field is unset.
+	Roots *x509.CertPool
+	// SPIFFEBundleMap is an in-memory representation of a spiffe trust bundle
+	// map. If this value exists, it will be used to find the roots for a given
+	// trust domain rather than the Roots in this struct.
+	SPIFFEBundleMap map[string]*spiffebundle.Bundle
+}
+
+// BuildOptions contains parameters passed to a Provider at build time.
+type BuildOptions struct {
+	// CertName holds the certificate name, whose key material is of interest to
+	// the caller.
+	CertName string
+	// WantRoot indicates if the caller is interested in the root certificate.
+	WantRoot bool
+	// WantIdentity indicates if the caller is interested in the identity
+	// certificate.
+	WantIdentity bool
+}

credentials/tls/certprovider/store.go

@@ -0,0 +1,196 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package certprovider
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+)
+
+// provStore is the global singleton certificate provider store.
+var provStore = &store{
+	providers: make(map[storeKey]*wrappedProvider),
+}
+
+// storeKey acts as the key to the map of providers maintained by the store. A
+// combination of provider name and configuration is used to uniquely identify
+// every provider instance in the store. Go maps need to be indexed by
+// comparable types, so the provider configuration is converted from `any` to
+// `string` using the ParseConfig method while creating this key.
+type storeKey struct {
+	// name of the certificate provider.
+	name string
+	// configuration of the certificate provider in string form.
+	config string
+	// opts contains the certificate name and other keyMaterial options.
+	opts BuildOptions
+}
+
+// wrappedProvider wraps a provider instance with a reference count.
+type wrappedProvider struct {
+	Provider
+	refCount int
+
+	// A reference to the key and store are also kept here to override the
+	// Close method on the provider.
+	storeKey storeKey
+	store    *store
+}
+
+// closedProvider always returns errProviderClosed error.
+type closedProvider struct{}
+
+func (c closedProvider) KeyMaterial(context.Context) (*KeyMaterial, error) {
+	return nil, errProviderClosed
+}
+
+func (c closedProvider) Close() {
+}
+
+// singleCloseWrappedProvider wraps a provider instance with a reference count
+// to properly handle multiple calls to Close.
+type singleCloseWrappedProvider struct {
+	provider atomic.Pointer[Provider]
+}
+
+// store is a collection of provider instances, safe for concurrent access.
+type store struct {
+	mu        sync.Mutex
+	providers map[storeKey]*wrappedProvider
+}
+
+// Close overrides the Close method of the embedded provider. It releases the
+// reference held by the caller on the underlying provider and if the
+// provider's reference count reaches zero, it is removed from the store, and
+// its Close method is also invoked.
+func (wp *wrappedProvider) Close() {
+	ps := wp.store
+	ps.mu.Lock()
+	defer ps.mu.Unlock()
+
+	wp.refCount--
+	if wp.refCount == 0 {
+		wp.Provider.Close()
+		delete(ps.providers, wp.storeKey)
+	}
+}
+
+// Close overrides the Close method of the embedded provider to avoid release the
+// already released reference.
+func (w *singleCloseWrappedProvider) Close() {
+	newProvider := Provider(closedProvider{})
+	oldProvider := w.provider.Swap(&newProvider)
+	(*oldProvider).Close()
+}
+
+// KeyMaterial returns the key material sourced by the Provider.
+// Callers are expected to use the returned value as read-only.
+func (w *singleCloseWrappedProvider) KeyMaterial(ctx context.Context) (*KeyMaterial, error) {
+	return (*w.provider.Load()).KeyMaterial(ctx)
+}
+
+// newSingleCloseWrappedProvider create wrapper a provider instance with a reference count
+// to properly handle multiple calls to Close.
+func newSingleCloseWrappedProvider(provider Provider) *singleCloseWrappedProvider {
+	w := &singleCloseWrappedProvider{}
+	w.provider.Store(&provider)
+	return w
+}
+
+// BuildableConfig wraps parsed provider configuration and functionality to
+// instantiate provider instances.
+type BuildableConfig struct {
+	name    string
+	config  []byte
+	starter func(BuildOptions) Provider
+	pStore  *store
+}
+
+// NewBuildableConfig creates a new BuildableConfig with the given arguments.
+// Provider implementations are expected to invoke this function after parsing
+// the given configuration as part of their ParseConfig() method.
+// Equivalent configurations are expected to invoke this function with the same
+// config argument.
+func NewBuildableConfig(name string, config []byte, starter func(BuildOptions) Provider) *BuildableConfig {
+	return &BuildableConfig{
+		name:    name,
+		config:  config,
+		starter: starter,
+		pStore:  provStore,
+	}
+}
+
+// Build kicks off a provider instance with the wrapped configuration. Multiple
+// invocations of this method with the same opts will result in provider
+// instances being reused.
+func (bc *BuildableConfig) Build(opts BuildOptions) (Provider, error) {
+	provStore.mu.Lock()
+	defer provStore.mu.Unlock()
+
+	sk := storeKey{
+		name:   bc.name,
+		config: string(bc.config),
+		opts:   opts,
+	}
+	if wp, ok := provStore.providers[sk]; ok {
+		wp.refCount++
+		return newSingleCloseWrappedProvider(wp), nil
+	}
+
+	provider := bc.starter(opts)
+	if provider == nil {
+		return nil, fmt.Errorf("provider(%q, %q).Build(%v) failed", sk.name, sk.config, opts)
+	}
+	wp := &wrappedProvider{
+		Provider: provider,
+		refCount: 1,
+		storeKey: sk,
+		store:    provStore,
+	}
+	provStore.providers[sk] = wp
+	return newSingleCloseWrappedProvider(wp), nil
+}
+
+// String returns the provider name and config as a colon separated string.
+func (bc *BuildableConfig) String() string {
+	return fmt.Sprintf("%s:%s", bc.name, string(bc.config))
+}
+
+// ParseConfig is a convenience function to create a BuildableConfig given a
+// provider name and configuration. Returns an error if there is no registered
+// builder for the given name or if the config parsing fails.
+func ParseConfig(name string, config any) (*BuildableConfig, error) {
+	parser := getBuilder(name)
+	if parser == nil {
+		return nil, fmt.Errorf("no certificate provider builder found for %q", name)
+	}
+	return parser.ParseConfig(config)
+}
+
+// GetProvider is a convenience function to create a provider given the name,
+// config and build options.
+func GetProvider(name string, config any, opts BuildOptions) (Provider, error) {
+	bc, err := ParseConfig(name, config)
+	if err != nil {
+		return nil, err
+	}
+	return bc.Build(opts)
+}

credentials/tls/certprovider/store_test.go

@@ -0,0 +1,400 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package certprovider
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/testdata"
+)
+
+const (
+	fakeProvider1Name       = "fake-certificate-provider-1"
+	fakeProvider2Name       = "fake-certificate-provider-2"
+	fakeConfig              = "my fake config"
+	defaultTestTimeout      = 5 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond
+)
+
+var fpb1, fpb2 *fakeProviderBuilder
+
+func init() {
+	fpb1 = &fakeProviderBuilder{
+		name:         fakeProvider1Name,
+		providerChan: testutils.NewChannel(),
+	}
+	fpb2 = &fakeProviderBuilder{
+		name:         fakeProvider2Name,
+		providerChan: testutils.NewChannel(),
+	}
+	Register(fpb1)
+	Register(fpb2)
+}
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// fakeProviderBuilder builds new instances of fakeProvider and interprets the
+// config provided to it as a string.
+type fakeProviderBuilder struct {
+	name         string
+	providerChan *testutils.Channel
+}
+
+func (b *fakeProviderBuilder) ParseConfig(config any) (*BuildableConfig, error) {
+	s, ok := config.(string)
+	if !ok {
+		return nil, fmt.Errorf("providerBuilder %s received config of type %T, want string", b.name, config)
+	}
+	return NewBuildableConfig(b.name, []byte(s), func(BuildOptions) Provider {
+		fp := &fakeProvider{
+			Distributor: NewDistributor(),
+			config:      s,
+		}
+		b.providerChan.Send(fp)
+		return fp
+	}), nil
+}
+
+func (b *fakeProviderBuilder) Name() string {
+	return b.name
+}
+
+// fakeProvider is an implementation of the Provider interface which provides a
+// method for tests to invoke to push new key materials.
+type fakeProvider struct {
+	*Distributor
+	config string
+}
+
+func (p *fakeProvider) Start(BuildOptions) Provider {
+	// This is practically a no-op since this provider doesn't do any work which
+	// needs to be started at this point.
+	return p
+}
+
+// newKeyMaterial allows tests to push new key material to the fake provider
+// which will be made available to users of this provider.
+func (p *fakeProvider) newKeyMaterial(km *KeyMaterial, err error) {
+	p.Distributor.Set(km, err)
+}
+
+// Close helps implement the Provider interface.
+func (p *fakeProvider) Close() {
+	p.Distributor.Stop()
+}
+
+// loadKeyMaterials is a helper to read cert/key files from testdata and convert
+// them into a KeyMaterialReader struct.
+func loadKeyMaterials(t *testing.T, cert, key, ca string) *KeyMaterial {
+	t.Helper()
+
+	certs, err := tls.LoadX509KeyPair(testdata.Path(cert), testdata.Path(key))
+	if err != nil {
+		t.Fatalf("Failed to load keyPair: %v", err)
+	}
+
+	pemData, err := os.ReadFile(testdata.Path(ca))
+	if err != nil {
+		t.Fatal(err)
+	}
+	roots := x509.NewCertPool()
+	roots.AppendCertsFromPEM(pemData)
+	return &KeyMaterial{Certs: []tls.Certificate{certs}, Roots: roots}
+}
+
+// kmReader wraps the KeyMaterial method exposed by Provider and Distributor
+// implementations. Defining the interface here makes it possible to use the
+// same helper from both provider and distributor tests.
+type kmReader interface {
+	KeyMaterial(context.Context) (*KeyMaterial, error)
+}
+
+// readAndVerifyKeyMaterial attempts to read key material from the given
+// provider and compares it against the expected key material.
+func readAndVerifyKeyMaterial(ctx context.Context, kmr kmReader, wantKM *KeyMaterial) error {
+	gotKM, err := kmr.KeyMaterial(ctx)
+	if err != nil {
+		return fmt.Errorf("KeyMaterial(ctx) failed: %w", err)
+	}
+	return compareKeyMaterial(gotKM, wantKM)
+}
+
+func compareKeyMaterial(got, want *KeyMaterial) error {
+	if len(got.Certs) != len(want.Certs) {
+		return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want)
+	}
+	for i := 0; i < len(got.Certs); i++ {
+		if !got.Certs[i].Leaf.Equal(want.Certs[i].Leaf) {
+			return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want)
+		}
+	}
+
+	if gotR, wantR := got.Roots, want.Roots; !gotR.Equal(wantR) {
+		return fmt.Errorf("keyMaterial roots = %v, want %v", gotR, wantR)
+	}
+
+	return nil
+}
+
+func createProvider(t *testing.T, name, config string, opts BuildOptions) Provider {
+	t.Helper()
+	prov, err := GetProvider(name, config, opts)
+	if err != nil {
+		t.Fatalf("GetProvider(%s, %s, %v) failed: %v", name, config, opts, err)
+	}
+	return prov
+}
+
+// TestStoreSingleProvider creates a single provider through the store and calls
+// methods on them.
+func (s) TestStoreSingleProvider(t *testing.T) {
+	prov := createProvider(t, fakeProvider1Name, fakeConfig, BuildOptions{CertName: "default"})
+	defer prov.Close()
+
+	// Our fakeProviderBuilder pushes newly created providers on a channel. Grab
+	// the fake provider from that channel.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	p, err := fpb1.providerChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
+	}
+	fakeProv := p.(*fakeProvider)
+
+	// Attempt to read from key material from the Provider returned by the
+	// store. This will fail because we have not pushed any key material into
+	// our fake provider.
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := readAndVerifyKeyMaterial(sCtx, prov, nil); !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatal(err)
+	}
+
+	// Load key material from testdata directory, push it into out fakeProvider
+	// and attempt to read from the Provider returned by the store.
+	testKM1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	fakeProv.newKeyMaterial(testKM1, nil)
+	if err := readAndVerifyKeyMaterial(ctx, prov, testKM1); err != nil {
+		t.Fatal(err)
+	}
+
+	// Push new key material and read from the Provider. This should returned
+	// updated key material.
+	testKM2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem")
+	fakeProv.newKeyMaterial(testKM2, nil)
+	if err := readAndVerifyKeyMaterial(ctx, prov, testKM2); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestStoreSingleProviderSameConfigDifferentOpts creates multiple providers of
+// same type, for same configs but different keyMaterial options through the
+// store (and expects the store's sharing mechanism to kick in) and calls
+// methods on them.
+func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) {
+	// Create three readers on the same fake provider. Two of these readers use
+	// certName `foo`, while the third one uses certName `bar`.
+	optsFoo := BuildOptions{CertName: "foo"}
+	provFoo1 := createProvider(t, fakeProvider1Name, fakeConfig, optsFoo)
+	provFoo2 := createProvider(t, fakeProvider1Name, fakeConfig, optsFoo)
+	defer func() {
+		provFoo1.Close()
+		provFoo2.Close()
+	}()
+
+	// Our fakeProviderBuilder pushes newly created providers on a channel.
+	// Grab the fake provider for optsFoo.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	p, err := fpb1.providerChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
+	}
+	fakeProvFoo := p.(*fakeProvider)
+
+	// Make sure only provider was created by the builder so far. The store
+	// should be able to share the providers.
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := fpb1.providerChan.Receive(sCtx); !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("A second provider created when expected to be shared by the store")
+	}
+
+	optsBar := BuildOptions{CertName: "bar"}
+	provBar1 := createProvider(t, fakeProvider1Name, fakeConfig, optsBar)
+	defer provBar1.Close()
+
+	// Grab the fake provider for optsBar.
+	p, err = fpb1.providerChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
+	}
+	fakeProvBar := p.(*fakeProvider)
+
+	// Push key material for optsFoo, and make sure the foo providers return
+	// appropriate key material and the bar provider times out.
+	fooKM := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	fakeProvFoo.newKeyMaterial(fooKM, nil)
+	if err := readAndVerifyKeyMaterial(ctx, provFoo1, fooKM); err != nil {
+		t.Fatal(err)
+	}
+	if err := readAndVerifyKeyMaterial(ctx, provFoo2, fooKM); err != nil {
+		t.Fatal(err)
+	}
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if err := readAndVerifyKeyMaterial(sCtx, provBar1, nil); !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatal(err)
+	}
+
+	// Push key material for optsBar, and make sure the bar provider returns
+	// appropriate key material.
+	barKM := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem")
+	fakeProvBar.newKeyMaterial(barKM, nil)
+	if err := readAndVerifyKeyMaterial(ctx, provBar1, barKM); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make sure the above push of new key material does not affect foo readers.
+	if err := readAndVerifyKeyMaterial(ctx, provFoo1, fooKM); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestStoreSingleProviderDifferentConfigs creates multiple instances of the
+// same type of provider through the store with different configs. The store
+// would end up creating different provider instances for these and no sharing
+// would take place.
+func (s) TestStoreSingleProviderDifferentConfigs(t *testing.T) {
+	// Create two providers of the same type, but with different configs.
+	opts := BuildOptions{CertName: "foo"}
+	cfg1 := fakeConfig + "1111"
+	cfg2 := fakeConfig + "2222"
+
+	prov1 := createProvider(t, fakeProvider1Name, cfg1, opts)
+	defer prov1.Close()
+	// Our fakeProviderBuilder pushes newly created providers on a channel. Grab
+	// the fake provider from that channel.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	p1, err := fpb1.providerChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
+	}
+	fakeProv1 := p1.(*fakeProvider)
+
+	prov2 := createProvider(t, fakeProvider1Name, cfg2, opts)
+	defer prov2.Close()
+	// Grab the second provider from the channel.
+	p2, err := fpb1.providerChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
+	}
+	fakeProv2 := p2.(*fakeProvider)
+
+	// Push the same key material into both fake providers and verify that the
+	// providers returned by the store return the appropriate key material.
+	km1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	fakeProv1.newKeyMaterial(km1, nil)
+	fakeProv2.newKeyMaterial(km1, nil)
+	if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil {
+		t.Fatal(err)
+	}
+	if err := readAndVerifyKeyMaterial(ctx, prov2, km1); err != nil {
+		t.Fatal(err)
+	}
+
+	// Push new key material into only one of the fake providers and verify
+	// that the providers returned by the store return the appropriate key
+	// material.
+	km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem")
+	fakeProv2.newKeyMaterial(km2, nil)
+	if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil {
+		t.Fatal(err)
+	}
+	if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil {
+		t.Fatal(err)
+	}
+
+	// Close one of the providers and verify that the other one is not affected.
+	prov1.Close()
+	if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestStoreMultipleProviders creates providers of different types and makes
+// sure closing of one does not affect the other.
+func (s) TestStoreMultipleProviders(t *testing.T) {
+	opts := BuildOptions{CertName: "foo"}
+	prov1 := createProvider(t, fakeProvider1Name, fakeConfig, opts)
+	defer prov1.Close()
+	// Our fakeProviderBuilder pushes newly created providers on a channel. Grab
+	// the fake provider from that channel.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	p1, err := fpb1.providerChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name)
+	}
+	fakeProv1 := p1.(*fakeProvider)
+
+	prov2 := createProvider(t, fakeProvider2Name, fakeConfig, opts)
+	defer prov2.Close()
+	// Grab the second provider from the channel.
+	p2, err := fpb2.providerChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider2Name)
+	}
+	fakeProv2 := p2.(*fakeProvider)
+
+	// Push the key material into both providers and verify that the
+	// readers return the appropriate key material.
+	km1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem")
+	fakeProv1.newKeyMaterial(km1, nil)
+	km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem")
+	fakeProv2.newKeyMaterial(km2, nil)
+	if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil {
+		t.Fatal(err)
+	}
+	if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil {
+		t.Fatal(err)
+	}
+
+	// Close one of the providers and verify that the other one is not affected.
+	prov1.Close()
+	if err := readAndVerifyKeyMaterial(ctx, prov2, km2); err != nil {
+		t.Fatal(err)
+	}
+}