snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)
Some checks failed
codecov / upload (push) Failing after 13m59s
CodeQL / Analyze (push) Failing after 1s
govulncheck / govulncheck (.) (push) Failing after 1s
govulncheck / govulncheck (./cmd/protoc-gen-go-grpc) (push) Failing after 1s
govulncheck / govulncheck (./gcp/observability) (push) Failing after 1s
govulncheck / govulncheck (./security/advancedtls) (push) Failing after 1s
govulncheck / govulncheck (./stats/opencensus) (push) Failing after 1s
Testing / tests (arm, latest) (push) Waiting to run
Testing / vet-proto (push) Failing after 1s
Testing / extras (latest) (push) Failing after 1s
Testing / static checks (latest-1) (push) Failing after 1s
Testing / tests (-race, latest) (push) Failing after 1s
Testing / tests (i386, latest) (push) Failing after 1s
Testing / tests (latest) (push) Failing after 1s
Testing / tests (latest-1) (push) Failing after 1s
Stale bot / stale (push) Failing after 1s
Lock Threads / lock (push) Failing after 1s

This commit is contained in:
upstream-mirror
2026-09-15 20:40:15 +05:00
commit e241b33f26
1365 changed files with 339570 additions and 0 deletions

View File

@@ -0,0 +1,36 @@
/*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient
import "google.golang.org/grpc/resolver"
type clientKeyType string
const clientKey = clientKeyType("grpc.xds.internal.client.Client")
// FromResolverState returns the Client from state, or nil if not present.
func FromResolverState(state resolver.State) XDSClient {
cs, _ := state.Attributes.Value(clientKey).(XDSClient)
return cs
}
// SetClient sets c in state and returns the new state.
func SetClient(state resolver.State, c XDSClient) resolver.State {
state.Attributes = state.Attributes.WithValue(clientKey, c)
return state
}

View File

@@ -0,0 +1,62 @@
/*
*
* Copyright 2019 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
// Package xdsclient implements a full fledged gRPC client for the xDS API used
// by the xds resolver and balancer implementations.
package xdsclient
import (
"context"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/clients/lrsclient"
"google.golang.org/grpc/internal/xds/clients/xdsclient"
v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
)
// XDSClient is a full fledged gRPC client which queries a set of discovery APIs
// (collectively termed as xDS) on a remote management server, to discover
// various dynamic resources.
type XDSClient interface {
// WatchResource uses xDS to discover the resource associated with the
// provided resource name. The resource type implementation determines how
// xDS responses are are deserialized and validated, as received from the
// xDS management server. Upon receipt of a response from the management
// server, an appropriate callback on the watcher is invoked.
//
// Most callers will not have a need to use this API directly. They will
// instead use a resource-type-specific wrapper API provided by the relevant
// resource type implementation.
//
//
// During a race (e.g. an xDS response is received while the user is calling
// cancel()), there's a small window where the callback can be called after
// the watcher is canceled. Callers need to handle this case.
WatchResource(typeURL, resourceName string, watcher xdsclient.ResourceWatcher) (cancel func())
ReportLoad(*bootstrap.ServerConfig) (*lrsclient.LoadStore, func(context.Context))
BootstrapConfig() *bootstrap.Config
}
// DumpResources returns the status and contents of all xDS resources. It uses
// xDS clients from the default pool.
func DumpResources() *v3statuspb.ClientStatusResponse {
return DefaultPool.DumpResources()
}

View File

@@ -0,0 +1,277 @@
/*
*
* Copyright 2022 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient
import (
"context"
"sync"
"testing"
"github.com/google/uuid"
estats "google.golang.org/grpc/experimental/stats"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/testutils/xds/e2e"
"google.golang.org/grpc/internal/xds/bootstrap"
)
// Tests that multiple calls to New() with the same name returns the same
// client. Also verifies that only when all references to the newly created
// client are released, the underlying client is closed.
func (s) TestClientNew_Single(t *testing.T) {
// Create a bootstrap configuration, place it in a file in the temp
// directory, and set the bootstrap env vars to point to it.
nodeID := uuid.New().String()
contents := e2e.DefaultBootstrapContents(t, nodeID, "non-existent-server-address")
config, err := bootstrap.NewConfigFromContents(contents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", contents, err)
}
pool := NewPool(config)
// Override the client creation hook to get notified.
origClientImplCreateHook := xdsClientImplCreateHook
clientImplCreateCh := testutils.NewChannel()
xdsClientImplCreateHook = func(name string) {
clientImplCreateCh.Replace(name)
}
defer func() { xdsClientImplCreateHook = origClientImplCreateHook }()
// Override the client close hook to get notified.
origClientImplCloseHook := xdsClientImplCloseHook
clientImplCloseCh := testutils.NewChannel()
xdsClientImplCloseHook = func(name string) {
clientImplCloseCh.Replace(name)
}
defer func() { xdsClientImplCloseHook = origClientImplCloseHook }()
// The first call to New() should create a new client.
_, closeFunc, err := pool.NewClient(t.Name(), &estats.UnimplementedMetricsRecorder{})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if _, err := clientImplCreateCh.Receive(ctx); err != nil {
t.Fatalf("Timeout when waiting for xDS client to be created: %v", err)
}
// Calling New() again should not create new client implementations.
const count = 9
closeFuncs := make([]func(), count)
for i := 0; i < count; i++ {
func() {
_, closeFuncs[i], err = pool.NewClient(t.Name(), &estats.UnimplementedMetricsRecorder{})
if err != nil {
t.Fatalf("%d-th call to New() failed with error: %v", i, err)
}
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
if _, err := clientImplCreateCh.Receive(sCtx); err == nil {
t.Fatalf("%d-th call to New() created a new client", i)
}
}()
}
// Call Close() multiple times on each of the clients created in the above
// for loop. Close() calls are idempotent, and the underlying client
// implementation will not be closed until we release the first reference we
// acquired above, via the first call to New().
for i := 0; i < count; i++ {
func() {
closeFuncs[i]()
closeFuncs[i]()
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
if _, err := clientImplCloseCh.Receive(sCtx); err == nil {
t.Fatal("Client implementation closed before all references are released")
}
}()
}
// Call the last Close(). The underlying implementation should be closed.
closeFunc()
if _, err := clientImplCloseCh.Receive(ctx); err != nil {
t.Fatalf("Timeout waiting for client implementation to be closed: %v", err)
}
// Calling New() again, after the previous Client was actually closed,
// should create a new one.
_, closeFunc, err = pool.NewClient(t.Name(), &estats.UnimplementedMetricsRecorder{})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
defer closeFunc()
if _, err := clientImplCreateCh.Receive(ctx); err != nil {
t.Fatalf("Timeout when waiting for xDS client to be created: %v", err)
}
}
// Tests the scenario where there are multiple calls to New() with different
// names. Verifies that reference counts are tracked correctly for each client
// and that only when all references are released for a client, it is closed.
func (s) TestClientNew_Multiple(t *testing.T) {
// Create a bootstrap configuration, place it in a file in the temp
// directory, and set the bootstrap env vars to point to it.
nodeID := uuid.New().String()
contents := e2e.DefaultBootstrapContents(t, nodeID, "non-existent-server-address")
config, err := bootstrap.NewConfigFromContents(contents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", contents, err)
}
pool := NewPool(config)
// Override the client creation hook to get notified.
origClientImplCreateHook := xdsClientImplCreateHook
clientImplCreateCh := testutils.NewChannel()
xdsClientImplCreateHook = func(name string) {
clientImplCreateCh.Replace(name)
}
defer func() { xdsClientImplCreateHook = origClientImplCreateHook }()
// Override the client close hook to get notified.
origClientImplCloseHook := xdsClientImplCloseHook
clientImplCloseCh := testutils.NewChannel()
xdsClientImplCloseHook = func(name string) {
clientImplCloseCh.Replace(name)
}
defer func() { xdsClientImplCloseHook = origClientImplCloseHook }()
// Create two xDS clients.
client1Name := t.Name() + "-1"
_, closeFunc1, err := pool.NewClient(client1Name, &estats.UnimplementedMetricsRecorder{})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
name, err := clientImplCreateCh.Receive(ctx)
if err != nil {
t.Fatalf("Timeout when waiting for xDS client to be created: %v", err)
}
if name.(string) != client1Name {
t.Fatalf("xDS client created for name %q, want %q", name.(string), client1Name)
}
client2Name := t.Name() + "-2"
_, closeFunc2, err := pool.NewClient(client2Name, &estats.UnimplementedMetricsRecorder{})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
name, err = clientImplCreateCh.Receive(ctx)
if err != nil {
t.Fatalf("Timeout when waiting for xDS client to be created: %v", err)
}
if name.(string) != client2Name {
t.Fatalf("xDS client created for name %q, want %q", name.(string), client1Name)
}
// Create N more references to each of these clients.
const count = 9
closeFuncs1 := make([]func(), count)
closeFuncs2 := make([]func(), count)
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
for i := 0; i < count; i++ {
var err error
_, closeFuncs1[i], err = pool.NewClient(client1Name, &estats.UnimplementedMetricsRecorder{})
if err != nil {
t.Errorf("%d-th call to New() failed with error: %v", i, err)
}
}
}()
go func() {
defer wg.Done()
for i := 0; i < count; i++ {
var err error
_, closeFuncs2[i], err = pool.NewClient(client2Name, &estats.UnimplementedMetricsRecorder{})
if err != nil {
t.Errorf("%d-th call to New() failed with error: %v", i, err)
}
}
}()
wg.Wait()
if t.Failed() {
t.FailNow()
}
// Ensure that none of the create hooks are called.
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
if _, err := clientImplCreateCh.Receive(sCtx); err == nil {
t.Fatalf("New xDS client created when expected to reuse an existing one")
}
// The close function returned by New() is idempotent and calling it
// multiple times should not decrement the reference count multiple times.
for i := 0; i < count; i++ {
closeFuncs1[i]()
closeFuncs1[i]()
}
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
if _, err := clientImplCloseCh.Receive(sCtx); err == nil {
t.Fatal("Client implementation closed before all references are released")
}
// Release the last reference and verify that the client is closed
// completely.
closeFunc1()
name, err = clientImplCloseCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for xDS client to be closed completely")
}
if name.(string) != client1Name {
t.Fatalf("xDS client closed for name %q, want %q", name.(string), client1Name)
}
// Ensure that the close hook is not called for the second client.
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
if _, err := clientImplCloseCh.Receive(sCtx); err == nil {
t.Fatal("Client implementation closed before all references are released")
}
// The close function returned by New() is idempotent and calling it
// multiple times should not decrement the reference count multiple times.
for i := 0; i < count; i++ {
closeFuncs2[i]()
closeFuncs2[i]()
}
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
if _, err := clientImplCloseCh.Receive(sCtx); err == nil {
t.Fatal("Client implementation closed before all references are released")
}
// Release the last reference and verify that the client is closed
// completely.
closeFunc2()
name, err = clientImplCloseCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for xDS client to be closed completely")
}
if name.(string) != client2Name {
t.Fatalf("xDS client closed for name %q, want %q", name.(string), client2Name)
}
}

View File

@@ -0,0 +1,40 @@
/*
*
* Copyright 2019 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient
import (
"testing"
"time"
"google.golang.org/grpc/internal/grpctest"
)
type s struct {
grpctest.Tester
}
func Test(t *testing.T) {
grpctest.RunSubTests(t, s{})
}
const (
defaultTestWatchExpiryTimeout = 100 * time.Millisecond
defaultTestTimeout = 5 * time.Second
defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen.
)

View File

@@ -0,0 +1,328 @@
/*
*
* Copyright 2022 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient
import (
"fmt"
"sync/atomic"
"time"
"google.golang.org/grpc"
estats "google.golang.org/grpc/experimental/stats"
"google.golang.org/grpc/internal/backoff"
"google.golang.org/grpc/internal/grpclog"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/clients"
"google.golang.org/grpc/internal/xds/clients/grpctransport"
"google.golang.org/grpc/internal/xds/clients/lrsclient"
"google.golang.org/grpc/internal/xds/clients/xdsclient"
"google.golang.org/grpc/internal/xds/clients/xdsclient/metrics"
xdsbootstrap "google.golang.org/grpc/xds/bootstrap"
)
const (
// NameForServer represents the value to be passed as name when creating an xDS
// client from xDS-enabled gRPC servers. This is a well-known dedicated key
// value, and is defined in gRFC A71.
NameForServer = "#server"
defaultWatchExpiryTimeout = 15 * time.Second
)
var (
// The following functions are no-ops in the actual code, but can be
// overridden in tests to give them visibility into certain events.
xdsClientImplCreateHook = func(string) {}
xdsClientImplCloseHook = func(string) {}
defaultExponentialBackoff = backoff.DefaultExponential.Backoff
xdsClientResourceUpdatesValidMetric = estats.RegisterInt64Count(estats.MetricDescriptor{
Name: "grpc.xds_client.resource_updates_valid",
Description: "A counter of resources received that were considered valid. The counter will be incremented even for resources that have not changed.",
Unit: "{resource}",
Labels: []string{"grpc.target", "grpc.xds.server", "grpc.xds.resource_type"},
Default: false,
})
xdsClientResourceUpdatesInvalidMetric = estats.RegisterInt64Count(estats.MetricDescriptor{
Name: "grpc.xds_client.resource_updates_invalid",
Description: "A counter of resources received that were considered invalid.",
Unit: "{resource}",
Labels: []string{"grpc.target", "grpc.xds.server", "grpc.xds.resource_type"},
Default: false,
})
xdsClientServerFailureMetric = estats.RegisterInt64Count(estats.MetricDescriptor{
Name: "grpc.xds_client.server_failure",
Description: "A counter of xDS servers going from healthy to unhealthy. A server goes unhealthy when we have a connectivity failure or when the ADS stream fails without seeing a response message, as per gRFC A57.",
Unit: "{failure}",
Labels: []string{"grpc.target", "grpc.xds.server"},
Default: false,
})
xdsClientConnectedMetric = estats.RegisterInt64AsyncGauge(estats.MetricDescriptor{
Name: "grpc.xds_client.connected",
Description: "A metric that is 1 if the xDS Client has a working ADS stream to the server, 0 otherwise.",
Unit: "{connected}",
Type: estats.MetricTypeIntAsyncGauge,
Labels: []string{"grpc.target", "grpc.xds.server"},
})
xdsClientResourcesMetric = estats.RegisterInt64AsyncGauge(estats.MetricDescriptor{
Name: "grpc.xds_client.resources",
Description: "Counts of xDS resources.",
Unit: "{resource}",
Type: estats.MetricTypeIntAsyncGauge,
Labels: []string{"grpc.target", "grpc.xds.authority", "grpc.xds.cache_state", "grpc.xds.resource_type"},
})
)
// clientImpl embed xdsclient.XDSClient and implement internal XDSClient
// interface with ref counting so that it can be shared by the xds resolver and
// balancer implementations, across multiple ClientConns and Servers.
type clientImpl struct {
*xdsclient.XDSClient // TODO: #8313 - get rid of embedding, if possible.
// The following fields are initialized at creation time and are read-only
// after that.
xdsClientConfig xdsclient.Config
bootstrapConfig *bootstrap.Config
logger *grpclog.PrefixLogger
target string
lrsClient *lrsclient.LRSClient
// Accessed atomically
refCount int32
}
// metricsReporter implements the clients.MetricsReporter interface and uses an
// underlying stats.MetricsRecorderList to record metrics.
type metricsReporter struct {
recorder estats.MetricsRecorder
target string
}
// ReportMetric implements the clients.MetricsReporter interface.
// It receives metric data, determines the appropriate metric based on the type
// of the data, and records it using the embedded MetricsRecorderList.
func (mr *metricsReporter) ReportMetric(metric any) {
if mr.recorder == nil {
return
}
switch m := metric.(type) {
case *metrics.ResourceUpdateValid:
xdsClientResourceUpdatesValidMetric.Record(mr.recorder, 1, mr.target, m.ServerURI, m.ResourceType)
case *metrics.ResourceUpdateInvalid:
xdsClientResourceUpdatesInvalidMetric.Record(mr.recorder, 1, mr.target, m.ServerURI, m.ResourceType)
case *metrics.ServerFailure:
xdsClientServerFailureMetric.Record(mr.recorder, 1, mr.target, m.ServerURI)
}
}
func newClientImpl(config *bootstrap.Config, metricsRecorder estats.MetricsRecorder, target string, watchExpiryTimeout time.Duration) (*clientImpl, error) {
gConfig, err := BuildXDSClientConfig(config, metricsRecorder, target, watchExpiryTimeout)
if err != nil {
return nil, err
}
client, err := xdsclient.New(gConfig)
if err != nil {
return nil, err
}
lrsC, err := lrsclient.New(lrsclient.Config{
Node: gConfig.Node,
TransportBuilder: gConfig.TransportBuilder,
})
if err != nil {
return nil, err
}
c := &clientImpl{
XDSClient: client,
xdsClientConfig: gConfig,
bootstrapConfig: config,
target: target,
refCount: 1,
lrsClient: lrsC,
}
c.logger = prefixLogger(c)
return c, nil
}
// BootstrapConfig returns the configuration read from the bootstrap file.
// Callers must treat the return value as read-only.
func (c *clientImpl) BootstrapConfig() *bootstrap.Config {
return c.bootstrapConfig
}
func (c *clientImpl) incrRef() int32 {
return atomic.AddInt32(&c.refCount, 1)
}
func (c *clientImpl) decrRef() int32 {
return atomic.AddInt32(&c.refCount, -1)
}
func buildServerConfigs(bootstrapSC []*bootstrap.ServerConfig, grpcTransportConfigs map[string]grpctransport.Config, gServerCfgMap map[xdsclient.ServerConfig]*bootstrap.ServerConfig) ([]xdsclient.ServerConfig, error) {
var gServerCfg []xdsclient.ServerConfig
for _, sc := range bootstrapSC {
if err := populateGRPCTransportConfigsFromServerConfig(sc, grpcTransportConfigs); err != nil {
return nil, err
}
var serverFeatures xdsclient.ServerFeature
if sc.ServerFeaturesIgnoreResourceDeletion() {
serverFeatures = serverFeatures | xdsclient.ServerFeatureIgnoreResourceDeletion
}
if sc.ServerFeaturesTrustedXDSServer() {
serverFeatures = serverFeatures | xdsclient.ServerFeatureTrustedXDSServer
}
gsc := xdsclient.ServerConfig{
ServerIdentifier: clients.ServerIdentifier{
ServerURI: sc.ServerURI(),
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: sc.SelectedChannelCreds().Type},
},
ServerFeature: serverFeatures,
}
gServerCfg = append(gServerCfg, gsc)
gServerCfgMap[gsc] = sc
}
return gServerCfg, nil
}
// BuildXDSClientConfig builds the xdsclient.Config from the bootstrap.Config.
//
// This function is exported for fuzz testing purposes only.
func BuildXDSClientConfig(config *bootstrap.Config, metricsRecorder estats.MetricsRecorder, target string, watchExpiryTimeout time.Duration) (xdsclient.Config, error) {
grpcTransportConfigs := make(map[string]grpctransport.Config)
gServerCfgMap := make(map[xdsclient.ServerConfig]*bootstrap.ServerConfig)
gAuthorities := make(map[string]xdsclient.Authority)
for name, cfg := range config.Authorities() {
// If server configs are specified in the authorities map, use that.
// Else, use the top-level server configs.
serverCfg := config.XDSServers()
if len(cfg.XDSServers) >= 1 {
serverCfg = cfg.XDSServers
}
gsc, err := buildServerConfigs(serverCfg, grpcTransportConfigs, gServerCfgMap)
if err != nil {
return xdsclient.Config{}, err
}
gAuthorities[name] = xdsclient.Authority{XDSServers: gsc}
}
gServerCfgs, err := buildServerConfigs(config.XDSServers(), grpcTransportConfigs, gServerCfgMap)
if err != nil {
return xdsclient.Config{}, err
}
node := config.Node()
gNode := clients.Node{
ID: node.GetId(),
Cluster: node.GetCluster(),
Metadata: node.Metadata,
UserAgentName: node.UserAgentName,
UserAgentVersion: node.GetUserAgentVersion(),
}
if node.Locality != nil {
gNode.Locality = clients.Locality{
Region: node.Locality.Region,
Zone: node.Locality.Zone,
SubZone: node.Locality.SubZone,
}
}
return xdsclient.Config{
Authorities: gAuthorities,
Servers: gServerCfgs,
Node: gNode,
TransportBuilder: grpctransport.NewBuilder(grpcTransportConfigs),
ResourceTypes: supportedResourceTypes(config, gServerCfgMap),
MetricsReporter: &metricsReporter{recorder: metricsRecorder, target: target},
WatchExpiryTimeout: watchExpiryTimeout,
}, nil
}
// populateGRPCTransportConfigsFromServerConfig iterates through the channel
// credentials of the provided server configuration, builds credential bundles,
// and populates the grpctransport.Config map.
func populateGRPCTransportConfigsFromServerConfig(sc *bootstrap.ServerConfig, grpcTransportConfigs map[string]grpctransport.Config) error {
for _, cc := range sc.ChannelCreds() {
c := xdsbootstrap.GetChannelCredentials(cc.Type)
if c == nil {
continue
}
bundle, _, err := c.Build(cc.Config)
if err != nil {
return fmt.Errorf("xds: failed to build credentials bundle from bootstrap for %q: %v", cc.Type, err)
}
grpcTransportConfigs[cc.Type] = grpctransport.Config{
Credentials: bundle,
GRPCNewClient: func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
opts = append(opts, sc.DialOptions()...)
return grpc.NewClient(target, opts...)
},
}
}
return nil
}
// RegisterAsyncReporter adapts the generic clients.AsyncReporter to the
// estats.AsyncMetricReporter interface and registers it.
func (mr *metricsReporter) RegisterAsyncReporter(reporter clients.AsyncReporter) func() {
if mr.recorder == nil || reporter == nil {
return func() {}
}
// Define which metrics we intend to report for OTel registration.
descriptors := []estats.AsyncMetric{
xdsClientConnectedMetric,
xdsClientResourcesMetric,
}
// Create the callback wrapper.
// This function is invoked by the stats system during a scrape.
cbWrapper := func(rec estats.AsyncMetricsRecorder) error {
wrapper := &asyncMetricsRecorderAdapter{
target: mr.target,
delegate: rec,
}
reporter.Report(wrapper)
return nil
}
// Register with the underlying gRPC stats recorder.
return mr.recorder.RegisterAsyncReporter(estats.AsyncMetricReporterFunc(cbWrapper), descriptors...)
}
// asyncMetricsRecorderAdapter adapts estats.AsyncMetricsRecorder to clients.AsyncMetricsRecorder.
type asyncMetricsRecorderAdapter struct {
target string
delegate estats.AsyncMetricsRecorder
}
func (a *asyncMetricsRecorderAdapter) ReportMetric(metric any) {
switch m := metric.(type) {
case *metrics.XDSClientConnected:
// Record: grpc.xds_client.connected
// Labels: grpc.target, grpc.xds.server
a.delegate.RecordInt64AsyncGauge(xdsClientConnectedMetric, m.Value, a.target, m.ServerURI)
case *metrics.XDSClientResourceStats:
// Record: grpc.xds_client.resources
// Labels: grpc.target, grpc.xds.authority, grpc.xds.cache_state, grpc.xds.resource_type
a.delegate.RecordInt64AsyncGauge(xdsClientResourcesMetric, m.Count, a.target, m.Authority, m.CacheState, m.ResourceType)
}
}

View File

@@ -0,0 +1,49 @@
/*
*
* Copyright 2019 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsclient
import (
"context"
"sync"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/clients"
"google.golang.org/grpc/internal/xds/clients/grpctransport"
"google.golang.org/grpc/internal/xds/clients/lrsclient"
)
// ReportLoad starts a load reporting stream to the given server. All load
// reports to the same server share the LRS stream.
//
// It returns a lrsclient.LoadStore for the user to report loads.
func (c *clientImpl) ReportLoad(server *bootstrap.ServerConfig) (*lrsclient.LoadStore, func(context.Context)) {
load, err := c.lrsClient.ReportLoad(clients.ServerIdentifier{
ServerURI: server.ServerURI(),
Extensions: grpctransport.ServerIdentifierExtension{
ConfigName: server.SelectedChannelCreds().Type,
},
})
if err != nil {
c.logger.Warningf("Failed to create a load store to the management server to report load: %v", server, err)
return nil, func(context.Context) {}
}
var loadStop sync.Once
return load, func(ctx context.Context) {
loadStop.Do(func() { load.Stop(ctx) })
}
}

View File

@@ -0,0 +1,363 @@
/*
*
* Copyright 2025 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient
import (
"encoding/json"
"fmt"
"reflect"
"sync"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/internal/envconfig"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/testutils/stats"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/clients"
"google.golang.org/grpc/internal/xds/clients/grpctransport"
"google.golang.org/grpc/internal/xds/clients/xdsclient"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
"google.golang.org/protobuf/testing/protocmp"
)
const (
testXDSServerURL = "xds.example.com:8080"
testXDSServerURL2 = "xds.example.com:8081"
testNodeID = "test-node-id"
testClusterName = "test-cluster"
testUserAgentName = "test-ua-name"
testUserAgentVer = "test-ua-ver"
testLocalityRegion = "test-region"
testLocalityZone = "test-zone"
testLocalitySubZone = "test-sub-zone"
testTargetName = "test-target"
)
var (
testMetadataJSON, _ = json.Marshal(map[string]any{"foo": "bar", "baz": float64(1)})
)
func (s) TestBuildXDSClientConfig_Success(t *testing.T) {
tests := []struct {
name string
bootstrapContents []byte
wantXDSClientConfig func(bootstrapCfg *bootstrap.Config) xdsclient.Config
}{
{
name: "without authorities",
bootstrapContents: []byte(fmt.Sprintf(`{
"xds_servers": [{"server_uri": "%s", "channel_creds": [{"type": "insecure"}]}],
"node": {
"id": "%s", "cluster": "%s", "metadata": %s,
"locality": {"region": "%s", "zone": "%s", "sub_zone": "%s"},
"user_agent_name": "%s", "user_agent_version": "%s"
}
}`, testXDSServerURL, testNodeID, testClusterName, testMetadataJSON, testLocalityRegion, testLocalityZone, testLocalitySubZone, testUserAgentName, testUserAgentVer)),
wantXDSClientConfig: func(c *bootstrap.Config) xdsclient.Config {
node, serverCfg := c.Node(), c.XDSServers()[0]
expectedServer := xdsclient.ServerConfig{ServerIdentifier: clients.ServerIdentifier{ServerURI: serverCfg.ServerURI(), Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}}
gServerCfgMap := map[xdsclient.ServerConfig]*bootstrap.ServerConfig{expectedServer: serverCfg}
return xdsclient.Config{
Servers: []xdsclient.ServerConfig{expectedServer},
Node: clients.Node{ID: node.GetId(), Cluster: node.GetCluster(), Metadata: node.Metadata, Locality: clients.Locality{Region: node.Locality.Region, Zone: node.Locality.Zone, SubZone: node.Locality.SubZone}, UserAgentName: node.UserAgentName, UserAgentVersion: node.GetUserAgentVersion()},
Authorities: map[string]xdsclient.Authority{},
ResourceTypes: map[string]xdsclient.ResourceType{
version.V3ListenerURL: {TypeURL: version.V3ListenerURL, TypeName: xdsresource.ListenerResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewListenerResourceTypeDecoder(c, gServerCfgMap)},
version.V3RouteConfigURL: {TypeURL: version.V3RouteConfigURL, TypeName: xdsresource.RouteConfigTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewRouteConfigResourceTypeDecoder(c, gServerCfgMap)},
version.V3ClusterURL: {TypeURL: version.V3ClusterURL, TypeName: xdsresource.ClusterResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewClusterResourceTypeDecoder(c, gServerCfgMap)},
version.V3EndpointsURL: {TypeURL: version.V3EndpointsURL, TypeName: xdsresource.EndpointsResourceTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewEndpointsResourceTypeDecoder(c)},
},
MetricsReporter: &metricsReporter{recorder: stats.NewTestMetricsRecorder(), target: testTargetName},
TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{
"insecure": {
Credentials: insecure.NewBundle(),
GRPCNewClient: func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
opts = append(opts, serverCfg.DialOptions()...)
return grpc.NewClient(target, opts...)
}},
}),
}
},
},
{
name: "with authorities",
bootstrapContents: []byte(fmt.Sprintf(`{
"xds_servers": [{"server_uri": "%s", "channel_creds": [{"type": "insecure"}]}],
"node": {"id": "%s"},
"authorities": {
"auth1": {},
"auth2": {"xds_servers": [{"server_uri": "%s", "channel_creds": [{"type": "insecure"}]}]}
}
}`, testXDSServerURL, testNodeID, testXDSServerURL2)),
wantXDSClientConfig: func(c *bootstrap.Config) xdsclient.Config {
node := c.Node()
topLevelSCfg, auth2SCfg := c.XDSServers()[0], c.Authorities()["auth2"].XDSServers[0]
expTopLevelS := xdsclient.ServerConfig{ServerIdentifier: clients.ServerIdentifier{ServerURI: topLevelSCfg.ServerURI(), Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}}
expAuth2S := xdsclient.ServerConfig{ServerIdentifier: clients.ServerIdentifier{ServerURI: auth2SCfg.ServerURI(), Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}}
gServerCfgMap := map[xdsclient.ServerConfig]*bootstrap.ServerConfig{expTopLevelS: topLevelSCfg, expAuth2S: auth2SCfg}
return xdsclient.Config{
Servers: []xdsclient.ServerConfig{expTopLevelS},
Node: clients.Node{ID: node.GetId(), Cluster: node.GetCluster(), Metadata: node.Metadata, UserAgentName: node.UserAgentName, UserAgentVersion: node.GetUserAgentVersion()},
Authorities: map[string]xdsclient.Authority{"auth1": {XDSServers: []xdsclient.ServerConfig{expTopLevelS}}, "auth2": {XDSServers: []xdsclient.ServerConfig{expAuth2S}}},
ResourceTypes: map[string]xdsclient.ResourceType{
version.V3ListenerURL: {TypeURL: version.V3ListenerURL, TypeName: xdsresource.ListenerResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewListenerResourceTypeDecoder(c, gServerCfgMap)},
version.V3RouteConfigURL: {TypeURL: version.V3RouteConfigURL, TypeName: xdsresource.RouteConfigTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewRouteConfigResourceTypeDecoder(c, gServerCfgMap)},
version.V3ClusterURL: {TypeURL: version.V3ClusterURL, TypeName: xdsresource.ClusterResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewClusterResourceTypeDecoder(c, gServerCfgMap)},
version.V3EndpointsURL: {TypeURL: version.V3EndpointsURL, TypeName: xdsresource.EndpointsResourceTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewEndpointsResourceTypeDecoder(c)},
},
MetricsReporter: &metricsReporter{recorder: stats.NewTestMetricsRecorder(), target: testTargetName},
TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{
"insecure": {
Credentials: insecure.NewBundle(),
GRPCNewClient: func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
opts = append(opts, topLevelSCfg.DialOptions()...)
return grpc.NewClient(target, opts...)
}},
}),
}
},
},
{
name: "server features with ignore_resource_deletion",
bootstrapContents: []byte(fmt.Sprintf(`{
"xds_servers": [{"server_uri": "%s", "channel_creds": [{"type": "insecure"}], "server_features": ["ignore_resource_deletion"]}],
"node": {"id": "%s"}
}`, testXDSServerURL, testNodeID)),
wantXDSClientConfig: func(c *bootstrap.Config) xdsclient.Config {
node, serverCfg := c.Node(), c.XDSServers()[0]
serverFeature := xdsclient.ServerFeatureIgnoreResourceDeletion
expectedServer := xdsclient.ServerConfig{
ServerIdentifier: clients.ServerIdentifier{ServerURI: serverCfg.ServerURI(), Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}},
ServerFeature: serverFeature}
gServerCfgMap := map[xdsclient.ServerConfig]*bootstrap.ServerConfig{expectedServer: serverCfg}
return xdsclient.Config{
Servers: []xdsclient.ServerConfig{expectedServer},
Node: clients.Node{ID: node.GetId(), Cluster: node.GetCluster(), Metadata: node.Metadata, UserAgentName: node.UserAgentName, UserAgentVersion: node.GetUserAgentVersion()},
Authorities: map[string]xdsclient.Authority{},
ResourceTypes: map[string]xdsclient.ResourceType{
version.V3ListenerURL: {TypeURL: version.V3ListenerURL, TypeName: xdsresource.ListenerResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewListenerResourceTypeDecoder(c, gServerCfgMap)},
version.V3RouteConfigURL: {TypeURL: version.V3RouteConfigURL, TypeName: xdsresource.RouteConfigTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewRouteConfigResourceTypeDecoder(c, gServerCfgMap)},
version.V3ClusterURL: {TypeURL: version.V3ClusterURL, TypeName: xdsresource.ClusterResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewClusterResourceTypeDecoder(c, gServerCfgMap)},
version.V3EndpointsURL: {TypeURL: version.V3EndpointsURL, TypeName: xdsresource.EndpointsResourceTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewEndpointsResourceTypeDecoder(c)},
},
MetricsReporter: &metricsReporter{recorder: stats.NewTestMetricsRecorder(), target: testTargetName},
TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{
"insecure": {
Credentials: insecure.NewBundle(),
GRPCNewClient: func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
opts = append(opts, serverCfg.DialOptions()...)
return grpc.NewClient(target, opts...)
}},
}),
}
},
},
{
name: "server features with trusted_xds_server",
bootstrapContents: []byte(fmt.Sprintf(`{
"xds_servers": [{"server_uri": "%s", "channel_creds": [{"type": "insecure"}], "server_features": ["trusted_xds_server"]}],
"node": {"id": "%s"}
}`, testXDSServerURL, testNodeID)),
wantXDSClientConfig: func(c *bootstrap.Config) xdsclient.Config {
node, serverCfg := c.Node(), c.XDSServers()[0]
serverFeature := xdsclient.ServerFeatureTrustedXDSServer
expectedServer := xdsclient.ServerConfig{ServerIdentifier: clients.ServerIdentifier{ServerURI: serverCfg.ServerURI(), Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}},
ServerFeature: serverFeature}
gServerCfgMap := map[xdsclient.ServerConfig]*bootstrap.ServerConfig{expectedServer: serverCfg}
return xdsclient.Config{
Servers: []xdsclient.ServerConfig{expectedServer},
Node: clients.Node{ID: node.GetId(), Cluster: node.GetCluster(), Metadata: node.Metadata, UserAgentName: node.UserAgentName, UserAgentVersion: node.GetUserAgentVersion()},
Authorities: map[string]xdsclient.Authority{},
ResourceTypes: map[string]xdsclient.ResourceType{
version.V3ListenerURL: {TypeURL: version.V3ListenerURL, TypeName: xdsresource.ListenerResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewListenerResourceTypeDecoder(c, gServerCfgMap)},
version.V3RouteConfigURL: {TypeURL: version.V3RouteConfigURL, TypeName: xdsresource.RouteConfigTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewRouteConfigResourceTypeDecoder(c, gServerCfgMap)},
version.V3ClusterURL: {TypeURL: version.V3ClusterURL, TypeName: xdsresource.ClusterResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewClusterResourceTypeDecoder(c, gServerCfgMap)},
version.V3EndpointsURL: {TypeURL: version.V3EndpointsURL, TypeName: xdsresource.EndpointsResourceTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewEndpointsResourceTypeDecoder(c)},
},
MetricsReporter: &metricsReporter{recorder: stats.NewTestMetricsRecorder(), target: testTargetName},
TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{
"insecure": {
Credentials: insecure.NewBundle(),
GRPCNewClient: func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
opts = append(opts, serverCfg.DialOptions()...)
return grpc.NewClient(target, opts...)
}},
}),
}
},
},
{
name: "server features with ignore_resource_deletion and trusted_xds_server",
bootstrapContents: []byte(fmt.Sprintf(`{
"xds_servers": [{"server_uri": "%s", "channel_creds": [{"type": "insecure"}], "server_features": ["ignore_resource_deletion", "trusted_xds_server"]}],
"node": {"id": "%s"}
}`, testXDSServerURL, testNodeID)),
wantXDSClientConfig: func(c *bootstrap.Config) xdsclient.Config {
node, serverCfg := c.Node(), c.XDSServers()[0]
serverFeature := xdsclient.ServerFeatureTrustedXDSServer | xdsclient.ServerFeatureIgnoreResourceDeletion
expectedServer := xdsclient.ServerConfig{ServerIdentifier: clients.ServerIdentifier{ServerURI: serverCfg.ServerURI(), Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}},
ServerFeature: serverFeature}
gServerCfgMap := map[xdsclient.ServerConfig]*bootstrap.ServerConfig{expectedServer: serverCfg}
return xdsclient.Config{
Servers: []xdsclient.ServerConfig{expectedServer},
Node: clients.Node{ID: node.GetId(), Cluster: node.GetCluster(), Metadata: node.Metadata, UserAgentName: node.UserAgentName, UserAgentVersion: node.GetUserAgentVersion()},
Authorities: map[string]xdsclient.Authority{},
ResourceTypes: map[string]xdsclient.ResourceType{
version.V3ListenerURL: {TypeURL: version.V3ListenerURL, TypeName: xdsresource.ListenerResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewListenerResourceTypeDecoder(c, gServerCfgMap)},
version.V3RouteConfigURL: {TypeURL: version.V3RouteConfigURL, TypeName: xdsresource.RouteConfigTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewRouteConfigResourceTypeDecoder(c, gServerCfgMap)},
version.V3ClusterURL: {TypeURL: version.V3ClusterURL, TypeName: xdsresource.ClusterResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewClusterResourceTypeDecoder(c, gServerCfgMap)},
version.V3EndpointsURL: {TypeURL: version.V3EndpointsURL, TypeName: xdsresource.EndpointsResourceTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewEndpointsResourceTypeDecoder(c)},
},
MetricsReporter: &metricsReporter{recorder: stats.NewTestMetricsRecorder(), target: testTargetName},
TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{
"insecure": {
Credentials: insecure.NewBundle(),
GRPCNewClient: func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
opts = append(opts, serverCfg.DialOptions()...)
return grpc.NewClient(target, opts...)
}},
}),
}
},
},
{
name: "channel creds - unknown type skipped",
bootstrapContents: []byte(fmt.Sprintf(`{
"xds_servers": [{"server_uri": "%s", "channel_creds": [{"type": "unknown-type"}, {"type": "insecure"}]}],
"node": {"id": "%s"}
}`, testXDSServerURL, testNodeID)), // "insecure" is selected
wantXDSClientConfig: func(c *bootstrap.Config) xdsclient.Config {
node, serverCfg := c.Node(), c.XDSServers()[0] // SelectedCreds will be "insecure"
expectedServer := xdsclient.ServerConfig{ServerIdentifier: clients.ServerIdentifier{ServerURI: serverCfg.ServerURI(), Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}}
gServerCfgMap := map[xdsclient.ServerConfig]*bootstrap.ServerConfig{expectedServer: serverCfg}
return xdsclient.Config{
Servers: []xdsclient.ServerConfig{expectedServer},
Node: clients.Node{ID: node.GetId(), Cluster: node.GetCluster(), Metadata: node.Metadata, UserAgentName: node.UserAgentName, UserAgentVersion: node.GetUserAgentVersion()},
Authorities: map[string]xdsclient.Authority{},
ResourceTypes: map[string]xdsclient.ResourceType{
version.V3ListenerURL: {TypeURL: version.V3ListenerURL, TypeName: xdsresource.ListenerResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewListenerResourceTypeDecoder(c, gServerCfgMap)},
version.V3RouteConfigURL: {TypeURL: version.V3RouteConfigURL, TypeName: xdsresource.RouteConfigTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewRouteConfigResourceTypeDecoder(c, gServerCfgMap)},
version.V3ClusterURL: {TypeURL: version.V3ClusterURL, TypeName: xdsresource.ClusterResourceTypeName, AllResourcesRequiredInSotW: true, Decoder: xdsresource.NewClusterResourceTypeDecoder(c, gServerCfgMap)},
version.V3EndpointsURL: {TypeURL: version.V3EndpointsURL, TypeName: xdsresource.EndpointsResourceTypeName, AllResourcesRequiredInSotW: false, Decoder: xdsresource.NewEndpointsResourceTypeDecoder(c)},
},
MetricsReporter: &metricsReporter{recorder: stats.NewTestMetricsRecorder(), target: testTargetName},
TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{
"insecure": {
Credentials: insecure.NewBundle(),
GRPCNewClient: func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
opts = append(opts, serverCfg.DialOptions()...)
return grpc.NewClient(target, opts...)
}},
}),
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
bootstrapConfig, err := bootstrap.NewConfigFromContents(tt.bootstrapContents)
if err != nil {
t.Fatalf("Failed to create bootstrap config: %v", err)
}
gotCfg, err := BuildXDSClientConfig(bootstrapConfig, stats.NewTestMetricsRecorder(), testTargetName, 0)
if err != nil {
t.Fatalf("Failed to build XDSClientConfig: %v", err)
}
wantCfg := tt.wantXDSClientConfig(bootstrapConfig)
unexportedTypeOpts := cmpopts.IgnoreUnexported(clients.Node{}, grpctransport.Builder{})
ignoreTypeOpts := cmpopts.IgnoreTypes(sync.Mutex{})
resourceTypeCmpOpts := cmp.Comparer(func(a, b xdsclient.ResourceType) bool {
return a.TypeURL == b.TypeURL && a.TypeName == b.TypeName && a.AllResourcesRequiredInSotW == b.AllResourcesRequiredInSotW && reflect.TypeOf(a.Decoder) == reflect.TypeOf(b.Decoder)
})
metricsReporterCmpOpts := cmp.Comparer(func(a, b clients.MetricsReporter) bool {
if (a == nil) != (b == nil) {
return false
}
if a == nil { // Both are nil
return true
}
// Both are non-nil, compare type and target.
aConcrete, aOK := a.(*metricsReporter)
bConcrete, bOK := b.(*metricsReporter)
if !(aOK && bOK && aConcrete.target == bConcrete.target) {
return false
}
// Compare recorder by type.
if (aConcrete.recorder == nil) != (bConcrete.recorder == nil) {
return false
}
// If both are nil, recorder check passes. If both non-nil, check types.
return aConcrete.recorder == nil || reflect.TypeOf(aConcrete.recorder) == reflect.TypeOf(bConcrete.recorder)
})
transportBuilderCmpOpts := cmp.Comparer(func(a, b grpctransport.Config) bool {
// Compare Credentials by type
credsEqual := true
if (a.Credentials == nil) != (b.Credentials == nil) {
credsEqual = false
} else if a.Credentials != nil && reflect.TypeOf(a.Credentials) != reflect.TypeOf(b.Credentials) {
credsEqual = false
}
// Compare GRPCNewClient by nil-ness
newClientEqual := (a.GRPCNewClient == nil) == (b.GRPCNewClient == nil)
return credsEqual && newClientEqual
})
if diff := cmp.Diff(wantCfg, gotCfg, protocmp.Transform(), unexportedTypeOpts, ignoreTypeOpts, resourceTypeCmpOpts, metricsReporterCmpOpts, transportBuilderCmpOpts); diff != "" {
t.Errorf("buildXDSClientConfig() mismatch (-want +got):\n%s", diff)
}
})
}
}
func (s) TestServerConfigCallCredsIntegration(t *testing.T) {
testutils.SetEnvConfig(t, &envconfig.XDSBootstrapCallCredsEnabled, true)
// Test server config with both channel and call credentials.
serverConfigJSON := `{
"server_uri": "xds-server:443",
"channel_creds": [{"type": "tls", "config": {}}],
"call_creds": [
{
"type": "jwt_token_file",
"config": {"jwt_token_file": "/token.jwt"}
}
]
}`
var sc bootstrap.ServerConfig
if err := sc.UnmarshalJSON([]byte(serverConfigJSON)); err != nil {
t.Fatalf("Failed to unmarshal server config: %v", err)
}
// Verify call credentials are processed.
callCreds := sc.CallCredsConfigs()
if len(callCreds) != 1 {
t.Errorf("Got %d call credential configs, want 1", len(callCreds))
}
dialOpts := sc.DialOptions()
if len(dialOpts) != 1 {
t.Errorf("Got %d dial options, want 1", len(dialOpts))
}
}

View File

@@ -0,0 +1,33 @@
/*
*
* Copyright 2024 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Package internal contains functionality internal to the xdsclient package.
package internal
// The following vars can be overridden by tests.
var (
// GRPCNewClient returns a new gRPC Client.
GRPCNewClient any // func(string, ...grpc.DialOption) (*grpc.ClientConn, error)
// NewADSStream returns a new ADS stream.
NewADSStream any // func(context.Context, *grpc.ClientConn) (v3adsgrpc.AggregatedDiscoveryService_StreamAggregatedResourcesClient, error)
// ResourceWatchStateForTesting gets the watch state for the resource
// identified by the given resource type and resource name. Returns a
// non-nil error if there is no such resource being watched.
ResourceWatchStateForTesting any // func(xdsclient.XDSClient, xdsresource.Type, string) error
)

View File

@@ -0,0 +1,36 @@
/*
*
* Copyright 2020 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient
import (
"fmt"
"google.golang.org/grpc/grpclog"
internalgrpclog "google.golang.org/grpc/internal/grpclog"
)
var logger = grpclog.Component("xds")
func prefixLogger(p *clientImpl) *internalgrpclog.PrefixLogger {
return internalgrpclog.NewPrefixLogger(logger, clientPrefix(p))
}
func clientPrefix(p *clientImpl) string {
return fmt.Sprintf("[xds-client %p] ", p)
}

View File

@@ -0,0 +1,361 @@
/*
*
* Copyright 2025 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient
import (
"context"
"encoding/json"
"errors"
"fmt"
"testing"
"github.com/google/uuid"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/testutils/stats"
"google.golang.org/grpc/internal/testutils/xds/e2e"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
_ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter.
)
type noopListenerWatcher struct{}
func (noopListenerWatcher) ResourceChanged(_ *xdsresource.ListenerUpdate, onDone func()) {
onDone()
}
func (noopListenerWatcher) ResourceError(_ error, onDone func()) {
onDone()
}
func (noopListenerWatcher) AmbientError(_ error, onDone func()) {
onDone()
}
// TestResourceUpdateMetrics configures an xDS client, and a management server
// to send valid and invalid LDS updates, and verifies that the expected metrics
// for both good and bad updates are emitted.
func (s) TestResourceUpdateMetrics(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
tmr := stats.NewTestMetricsRecorder()
l, err := testutils.LocalTCPListener()
if err != nil {
t.Fatalf("net.Listen() failed: %v", err)
}
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: l})
const listenerResourceName = "test-listener-resource"
const routeConfigurationName = "test-route-configuration-resource"
nodeID := uuid.New().String()
resources := e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)},
SkipValidation: true,
}
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, mgmtServer.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
Authorities: map[string]json.RawMessage{
"authority": []byte("{}"),
},
})
if err != nil {
t.Fatalf("Failed to create bootstrap configuration: %v", err)
}
config, err := bootstrap.NewConfigFromContents(bootstrapContents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
}
pool := NewPool(config)
client, close, err := pool.NewClientForTesting(OptionsForTesting{
Name: t.Name(),
WatchExpiryTimeout: defaultTestWatchExpiryTimeout,
MetricsRecorder: tmr,
})
if err != nil {
t.Fatalf("Failed to create an xDS client: %v", err)
}
defer close()
// Watch the valid listener configured on the management server. This should
// cause a resource updates valid count to emit eventually.
xdsresource.WatchListener(client, listenerResourceName, noopListenerWatcher{})
mdWant := stats.MetricsData{
Handle: xdsClientResourceUpdatesValidMetric.Descriptor(),
IntIncr: 1,
LabelKeys: []string{"grpc.target", "grpc.xds.server", "grpc.xds.resource_type"},
LabelVals: []string{"Test/ResourceUpdateMetrics", mgmtServer.Address, "ListenerResource"},
}
if err := tmr.WaitForInt64Count(ctx, mdWant); err != nil {
t.Fatal(err.Error())
}
// Invalid should have no recording point.
if got, _ := tmr.Metric("grpc.xds_client.resource_updates_invalid"); got != 0 {
t.Fatalf("Unexpected data for metric \"grpc.xds_client.resource_updates_invalid\", got: %v, want: %v", got, 0)
}
// Update management server with a bad update. Eventually, tmr should
// receive an invalid count received metric. The successful metric should
// stay the same.
resources = e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)},
SkipValidation: true,
}
resources.Listeners[0].ApiListener = nil
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
mdWant = stats.MetricsData{
Handle: xdsClientResourceUpdatesInvalidMetric.Descriptor(),
IntIncr: 1,
LabelKeys: []string{"grpc.target", "grpc.xds.server", "grpc.xds.resource_type"},
LabelVals: []string{"Test/ResourceUpdateMetrics", mgmtServer.Address, "ListenerResource"},
}
if err := tmr.WaitForInt64Count(ctx, mdWant); err != nil {
t.Fatal(err.Error())
}
// Valid should stay the same at 1.
if got, _ := tmr.Metric("grpc.xds_client.resource_updates_valid"); got != 1 {
t.Fatalf("Unexpected data for metric \"grpc.xds_client.resource_updates_invalid\", got: %v, want: %v", got, 1)
}
}
// TestServerFailureMetrics_BeforeResponseRecv configures an xDS client, and a
// management server. It then register a watcher and stops the management
// server before sending a resource update, and verifies that the expected
// metrics for server failure are emitted.
func (s) TestServerFailureMetrics_BeforeResponseRecv(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
tmr := stats.NewTestMetricsRecorder()
l, err := testutils.LocalTCPListener()
if err != nil {
t.Fatalf("net.Listen() failed: %v", err)
}
lis := testutils.NewRestartableListener(l)
streamOpened := make(chan struct{}, 1)
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
Listener: lis,
OnStreamOpen: func(context.Context, int64, string) error {
select {
case streamOpened <- struct{}{}:
default:
}
return nil
},
})
nodeID := uuid.New().String()
bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, mgmtServer.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
Authorities: map[string]json.RawMessage{
"authority": []byte("{}"),
},
})
if err != nil {
t.Fatalf("Failed to create bootstrap configuration: %v", err)
}
config, err := bootstrap.NewConfigFromContents(bootstrapContents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
}
pool := NewPool(config)
client, close, err := pool.NewClientForTesting(OptionsForTesting{
Name: t.Name(),
WatchExpiryTimeout: defaultTestWatchExpiryTimeout,
MetricsRecorder: tmr,
})
if err != nil {
t.Fatalf("Failed to create an xDS client: %v", err)
}
defer close()
const listenerResourceName = "test-listener-resource"
// Watch for the listener on the above management server.
xdsresource.WatchListener(client, listenerResourceName, noopListenerWatcher{})
// Verify that an ADS stream is opened and an LDS request with the above
// resource name is sent.
select {
case <-streamOpened:
case <-ctx.Done():
t.Fatal("Timeout when waiting for ADS stream to open")
}
// Close the listener and ensure that the ADS stream breaks. This should
// cause a server failure count to emit eventually.
lis.Stop()
// Restart to prevent the attempt to create a new ADS stream after back off.
lis.Restart()
mdWant := stats.MetricsData{
Handle: xdsClientServerFailureMetric.Descriptor(),
IntIncr: 1,
LabelKeys: []string{"grpc.target", "grpc.xds.server"},
LabelVals: []string{"Test/ServerFailureMetrics_BeforeResponseRecv", mgmtServer.Address},
}
if err := tmr.WaitForInt64Count(ctx, mdWant); err != nil {
t.Fatal(err.Error())
}
}
// TestServerFailureMetrics_AfterResponseRecv configures an xDS client and a
// management server to send a valid LDS update, and verifies that the
// successful update metric is emitted. When the client ACKs the update, the
// server returns an error, breaking the stream. The test then verifies that the
// server failure metric is not emitted, because the ADS stream was closed after
// a response was received on the stream. Finally, the test waits for the client
// to establish a new stream and verifies that the client emits a metric after
// receiving a successful update.
func (s) TestServerFailureMetrics_AfterResponseRecv(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
tmr := stats.NewTestMetricsRecorder()
l, err := testutils.LocalTCPListener()
if err != nil {
t.Fatalf("net.Listen() failed: %v", err)
}
lis := testutils.NewRestartableListener(l)
streamCreationQuota := make(chan struct{}, 1)
streamCreationQuota <- struct{}{}
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
Listener: lis,
OnStreamOpen: func(context.Context, int64, string) error {
// The following select block is used to block stream creation after
// the first stream has failed, but while we are waiting to verify
// that the failure metric is not reported.
select {
case <-streamCreationQuota:
case <-ctx.Done():
}
return nil
},
OnStreamRequest: func(streamID int64, req *v3discoverypb.DiscoveryRequest) error {
// We only want the ACK on the first stream to return an error
// (leading to stream closure), without effecting subsequent stream
// attempts.
if streamID == 1 && req.GetVersionInfo() != "" {
return errors.New("test configured error")
}
return nil
}},
)
const listenerResourceName = "test-listener-resource"
const routeConfigurationName = "test-route-configuration-resource"
nodeID := uuid.New().String()
resources := e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)},
SkipValidation: true,
}
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, mgmtServer.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
Authorities: map[string]json.RawMessage{
"authority": []byte("{}"),
},
})
if err != nil {
t.Fatalf("Failed to create bootstrap configuration: %v", err)
}
config, err := bootstrap.NewConfigFromContents(bootstrapContents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
}
pool := NewPool(config)
client, closePool, err := pool.NewClientForTesting(OptionsForTesting{
Name: t.Name(),
MetricsRecorder: tmr,
})
if err != nil {
t.Fatalf("Failed to create an xDS client: %v", err)
}
defer closePool()
// Watch the valid listener configured on the management server. This should
// cause a resource updates valid count to emit eventually.
xdsresource.WatchListener(client, listenerResourceName, noopListenerWatcher{})
mdSuccess := stats.MetricsData{
Handle: xdsClientResourceUpdatesValidMetric.Descriptor(),
IntIncr: 1,
LabelKeys: []string{"grpc.target", "grpc.xds.server", "grpc.xds.resource_type"},
LabelVals: []string{"Test/ServerFailureMetrics_AfterResponseRecv", mgmtServer.Address, "ListenerResource"},
}
if err := tmr.WaitForInt64Count(ctx, mdSuccess); err != nil {
t.Fatal(err.Error())
}
// When the client sends an ACK, the management server would reply with an
// error, breaking the stream.
mdFailure := stats.MetricsData{
Handle: xdsClientServerFailureMetric.Descriptor(),
IntIncr: 1,
LabelKeys: []string{"grpc.target", "grpc.xds.server"},
LabelVals: []string{"Test/ServerFailureMetrics_AfterResponseRecv", mgmtServer.Address},
}
// Server failure should still have no recording point.
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
if err := tmr.WaitForInt64Count(sCtx, mdFailure); err == nil {
t.Fatalf("tmr.WaitForInt64Count(%v) succeeded when expected to timeout.", mdFailure)
} else if sCtx.Err() == nil {
t.Fatalf("tmr.WaitForInt64Count(%v) = %v, want context deadline exceeded", mdFailure, err)
}
// Unblock stream creation and verify that an update is received
// successfully.
close(streamCreationQuota)
if err := tmr.WaitForInt64Count(ctx, mdSuccess); err != nil {
t.Fatal(err.Error())
}
}

View File

@@ -0,0 +1,320 @@
/*
*
* Copyright 2024 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient
import (
"fmt"
"sync"
"time"
v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
estats "google.golang.org/grpc/experimental/stats"
"google.golang.org/grpc/internal/envconfig"
istats "google.golang.org/grpc/internal/stats"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/protobuf/proto"
)
var (
// DefaultPool is the default pool for xDS clients. It is created at init
// time and reads bootstrap configuration from env vars to create the xDS
// client.
DefaultPool = &Pool{
clients: make(map[string]*clientImpl),
getConfiguration: sync.OnceValues(bootstrap.GetConfiguration),
}
)
// Pool represents a pool of xDS clients that share the same bootstrap
// configuration.
type Pool struct {
// Note that mu should ideally only have to guard clients. But here, we need
// it to guard config as well since SetFallbackBootstrapConfig writes to
// config.
mu sync.Mutex
clients map[string]*clientImpl
fallbackConfig *bootstrap.Config // TODO(i/8661): remove fallbackConfig.
// getConfiguration is a sync.OnceValues that attempts to read the bootstrap
// configuration from environment variables once.
getConfiguration func() (*bootstrap.Config, error)
}
// OptionsForTesting contains options to configure xDS client creation for
// testing purposes only.
type OptionsForTesting struct {
// Name is a unique name for this xDS client.
Name string
// WatchExpiryTimeout is the timeout for xDS resource watch expiry. If
// unspecified, uses the default value used in non-test code.
WatchExpiryTimeout time.Duration
// StreamBackoffAfterFailure is the backoff function used to determine the
// backoff duration after stream failures.
// If unspecified, uses the default value used in non-test code.
StreamBackoffAfterFailure func(int) time.Duration
// MetricsRecorder is the metrics recorder the xDS Client will use. If
// unspecified, uses a no-op MetricsRecorder.
MetricsRecorder estats.MetricsRecorder
// Config is the xDS bootstrap configuration that will be used to initialize
// the client. If unset, the client will use the config provided by env
// variables.
Config *bootstrap.Config
}
// NewPool creates a new xDS client pool with the given bootstrap config.
//
// If a nil bootstrap config is passed and SetFallbackBootstrapConfig is not
// called before a call to NewClient, the latter will fail. i.e. if there is an
// attempt to create an xDS client from the pool without specifying bootstrap
// configuration (either at pool creation time or by setting the fallback
// bootstrap configuration), xDS client creation will fail.
func NewPool(config *bootstrap.Config) *Pool {
return &Pool{
clients: make(map[string]*clientImpl),
getConfiguration: func() (*bootstrap.Config, error) {
return config, nil
},
}
}
// NewClientWithConfig returns an xDS client with the given name from the pool. If the
// client doesn't already exist, it creates a new xDS client and adds it to the
// pool.
//
// The second return value represents a close function which the caller is
// expected to invoke once they are done using the client. It is safe for the
// caller to invoke this close function multiple times.
func (p *Pool) NewClientWithConfig(name string, metricsRecorder estats.MetricsRecorder, config *bootstrap.Config) (XDSClient, func(), error) {
return p.newRefCounted(name, metricsRecorder, defaultWatchExpiryTimeout, config)
}
// NewClient returns an xDS client with the given name from the pool. If the
// client doesn't already exist, it creates a new xDS client and adds it to the
// pool.
//
// The second return value represents a close function which the caller is
// expected to invoke once they are done using the client. It is safe for the
// caller to invoke this close function multiple times.
func (p *Pool) NewClient(name string, metricsRecorder estats.MetricsRecorder) (XDSClient, func(), error) {
return p.newRefCounted(name, metricsRecorder, defaultWatchExpiryTimeout, nil)
}
// NewClientForTesting returns an xDS client configured with the provided
// options from the pool. If the client doesn't already exist, it creates a new
// xDS client and adds it to the pool.
//
// The second return value represents a close function which the caller is
// expected to invoke once they are done using the client. It is safe for the
// caller to invoke this close function multiple times.
//
// # Testing Only
//
// This function should ONLY be used for testing purposes.
func (p *Pool) NewClientForTesting(opts OptionsForTesting) (XDSClient, func(), error) {
if opts.Name == "" {
return nil, nil, fmt.Errorf("xds: opts.Name field must be non-empty")
}
if opts.WatchExpiryTimeout == 0 {
opts.WatchExpiryTimeout = defaultWatchExpiryTimeout
}
if opts.StreamBackoffAfterFailure == nil {
opts.StreamBackoffAfterFailure = defaultExponentialBackoff
}
if opts.MetricsRecorder == nil {
opts.MetricsRecorder = istats.NewMetricsRecorderList(nil)
}
c, cancel, err := p.newRefCounted(opts.Name, opts.MetricsRecorder, opts.WatchExpiryTimeout, opts.Config)
if err != nil {
return nil, nil, err
}
return c, cancel, nil
}
// GetClientForTesting returns an xDS client created earlier using the given
// name from the pool. If the client with the given name doesn't already exist,
// it returns an error.
//
// The second return value represents a close function which the caller is
// expected to invoke once they are done using the client. It is safe for the
// caller to invoke this close function multiple times.
//
// # Testing Only
//
// This function should ONLY be used for testing purposes.
func (p *Pool) GetClientForTesting(name string) (XDSClient, func(), error) {
p.mu.Lock()
defer p.mu.Unlock()
c, ok := p.clients[name]
if !ok {
return nil, nil, fmt.Errorf("xds:: xDS client with name %q not found", name)
}
c.incrRef()
return c, sync.OnceFunc(func() { p.clientRefCountedClose(name) }), nil
}
// SetFallbackBootstrapConfig is used to specify a bootstrap configuration
// that will be used as a fallback when the bootstrap environment variables
// are not defined.
// TODO(i/8661): remove SetFallbackBootstrapConfig function.
func (p *Pool) SetFallbackBootstrapConfig(config *bootstrap.Config) {
p.mu.Lock()
defer p.mu.Unlock()
p.fallbackConfig = config
}
// DumpResources returns the status and contents of all xDS resources.
func (p *Pool) DumpResources() *v3statuspb.ClientStatusResponse {
p.mu.Lock()
defer p.mu.Unlock()
resp := &v3statuspb.ClientStatusResponse{}
for key, client := range p.clients {
b, err := client.DumpResources()
if err != nil {
return nil
}
r := &v3statuspb.ClientStatusResponse{}
if err := proto.Unmarshal(b, r); err != nil {
return nil
}
cfg := r.Config[0]
cfg.ClientScope = key
resp.Config = append(resp.Config, cfg)
}
return resp
}
// BootstrapConfigForTesting returns the bootstrap configuration used by the
// pool. The caller should not mutate the returned config.
//
// To be used only for testing purposes.
func (p *Pool) BootstrapConfigForTesting() *bootstrap.Config {
p.mu.Lock()
defer p.mu.Unlock()
cfg, _ := p.getConfiguration()
if cfg != nil {
return cfg
}
return p.fallbackConfig
}
// UnsetBootstrapConfigForTesting unsets the bootstrap configuration used by
// the pool.
//
// To be used only for testing purposes.
func (p *Pool) UnsetBootstrapConfigForTesting() {
p.mu.Lock()
defer p.mu.Unlock()
p.fallbackConfig = nil
p.getConfiguration = sync.OnceValues(bootstrap.GetConfiguration)
}
func (p *Pool) clientRefCountedClose(name string) {
p.mu.Lock()
client, ok := p.clients[name]
if !ok {
logger.Errorf("Attempt to close a non-existent xDS client with name %s", name)
p.mu.Unlock()
return
}
if client.decrRef() != 0 {
p.mu.Unlock()
return
}
delete(p.clients, name)
for _, s := range client.bootstrapConfig.XDSServers() {
for _, f := range s.Cleanups() {
f()
}
}
for _, a := range client.bootstrapConfig.Authorities() {
for _, s := range a.XDSServers {
for _, f := range s.Cleanups() {
f()
}
}
}
// The allowed-services credentials are owned by the (pool-shared)
// bootstrap config and are released when the last reference to the
// client is dropped, following the same lifecycle as the xDS server
// credentials released above.
for _, svc := range client.bootstrapConfig.AllowedGRPCServices() {
for _, f := range svc.Cleanups() {
f()
}
}
p.mu.Unlock()
// This attempts to close the transport to the management server and could
// theoretically call back into the xdsclient package again and deadlock.
// Hence, this needs to be called without holding the lock.
client.Close()
xdsClientImplCloseHook(name)
}
// newRefCounted creates a new reference counted xDS client implementation for
// name, if one does not exist already. If an xDS client for the given name
// exists, it gets a reference to it and returns it.
func (p *Pool) newRefCounted(name string, metricsRecorder estats.MetricsRecorder, watchExpiryTimeout time.Duration, bConfig *bootstrap.Config) (*clientImpl, func(), error) {
p.mu.Lock()
defer p.mu.Unlock()
if c := p.clients[name]; c != nil {
c.incrRef()
return c, sync.OnceFunc(func() { p.clientRefCountedClose(name) }), nil
}
config := bConfig
if config == nil {
var err error
config, err = p.getConfiguration()
if err != nil {
return nil, nil, fmt.Errorf("xds: failed to read xDS bootstrap config from env vars: %v", err)
}
if config == nil {
// If the environment variables are not set, then fallback bootstrap
// configuration should be set before attempting to create an xDS client,
// else xDS client creation will fail.
config = p.fallbackConfig
}
}
if config == nil {
return nil, nil, fmt.Errorf("failed to read xDS bootstrap config from env vars: bootstrap environment variables (%q or %q) not defined and fallback config not set", envconfig.XDSBootstrapFileNameEnv, envconfig.XDSBootstrapFileContentEnv)
}
c, err := newClientImpl(config, metricsRecorder, name, watchExpiryTimeout)
if err != nil {
return nil, nil, err
}
if logger.V(2) {
c.logger.Infof("Created client with name %q and bootstrap configuration:\n %s", name, config)
}
p.clients[name] = c
xdsClientImplCreateHook(name)
logger.Infof("xDS node ID: %s", config.Node().GetId())
return c, sync.OnceFunc(func() { p.clientRefCountedClose(name) }), nil
}

View File

@@ -0,0 +1,250 @@
/*
*
* Copyright 2025 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package pool_ext_test
import (
"context"
"encoding/json"
"fmt"
"testing"
"time"
"github.com/google/uuid"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/internal/grpctest"
"google.golang.org/grpc/internal/stubserver"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/testutils/xds/e2e"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/xdsclient"
_ "google.golang.org/grpc/xds"
v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
estats "google.golang.org/grpc/experimental/stats"
testgrpc "google.golang.org/grpc/interop/grpc_testing"
testpb "google.golang.org/grpc/interop/grpc_testing"
)
const defaultTestTimeout = 10 * time.Second
type s struct {
grpctest.Tester
}
func Test(t *testing.T) {
grpctest.RunSubTests(t, s{})
}
// TestDefaultPool_LazyLoadBootstrapConfig verifies that the DefaultPool
// lazily loads the bootstrap configuration from environment variables when
// an xDS client is created for the first time.
//
// If tries to create the new client in DefaultPool at the start of test when
// none of the env vars are set. This should fail.
//
// Then it sets the env var XDSBootstrapFileName and retry creating a client
// in DefaultPool. This should succeed.
func (s) TestDefaultPool_LazyLoadBootstrapConfig(t *testing.T) {
_, closeFunc, err := xdsclient.DefaultPool.NewClient(t.Name(), &estats.UnimplementedMetricsRecorder{})
if err == nil {
t.Fatalf("xdsclient.DefaultPool.NewClient() succeeded without setting bootstrap config env vars, want failure")
}
bs, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, "non-existent-management-server")),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, uuid.New().String())),
CertificateProviders: map[string]json.RawMessage{
"cert-provider-instance": json.RawMessage("{}"),
},
})
if err != nil {
t.Fatalf("Failed to create bootstrap configuration: %v", err)
}
testutils.CreateBootstrapFileForTesting(t, bs)
if cfg := xdsclient.DefaultPool.BootstrapConfigForTesting(); cfg != nil {
t.Fatalf("DefaultPool.BootstrapConfigForTesting() = %v, want nil", cfg)
}
// The pool will attempt to read the env vars only once. Reset the pool's
// state to make it re-read the env vars during next client creation.
xdsclient.DefaultPool.UnsetBootstrapConfigForTesting()
_, closeFunc, err = xdsclient.DefaultPool.NewClient(t.Name(), &estats.UnimplementedMetricsRecorder{})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
defer func() {
closeFunc()
xdsclient.DefaultPool.UnsetBootstrapConfigForTesting()
}()
if xdsclient.DefaultPool.BootstrapConfigForTesting() == nil {
t.Fatalf("DefaultPool.BootstrapConfigForTesting() = nil, want non-nil")
}
}
// TestNestedXDSChannel tests a scenario where xDS is used to resolve the
// address of the management server, resulting in the creation of two nested xDS
// client channels. The server URI of the first xDS management server uses
// the xDS scheme. To resolve its address, the client creates a second xDS
// channel to a different management server. After retrieving the EDS resource
// for the first management server, the client connects to it to obtain the
// address of the test stub server. The test verifies that an RPC to the test
// stub server succeeds. It also helps detect potential deadlocks during the
// creation and teardown of the two nested xDS channels.
func (s) TestNestedXDSChannel(t *testing.T) {
lis1, err := testutils.LocalTCPListener()
if err != nil {
t.Fatal(err)
}
lis2, err := testutils.LocalTCPListener()
if err != nil {
t.Fatal(err)
}
const managementServerServiceName = "management_server"
const managementServerAuthority = "management_server_authority"
managementServerLDSName := fmt.Sprintf(
"xdstp://%s/envoy.config.listener.v3.Listener/%s",
managementServerAuthority, managementServerServiceName,
)
managementServerRDSName := fmt.Sprintf(
"xdstp://%s/envoy.config.route.v3.RouteConfiguration/route-%s",
managementServerAuthority, managementServerServiceName,
)
managementServerCDSName := fmt.Sprintf(
"xdstp://%s/envoy.config.cluster.v3.Cluster/cluster-%s",
managementServerAuthority, managementServerServiceName,
)
managementServerEDSName := fmt.Sprintf(
"xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/endpoints-%s",
managementServerAuthority, managementServerServiceName,
)
const serviceName = "my-service-client-side-xds"
const rdsName = "route-" + serviceName
const cdsName = "cluster-" + serviceName
const edsName = "endpoints-" + serviceName
// Start a management server that stores resources related to the test
// server.
managementServer1 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lis1})
// Start a management server that stores resources related to
// managementServer1.
managementServer2 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lis2})
server := stubserver.StartTestService(t, nil)
defer server.Stop()
// Create a bootstrap configuration where the xDS scheme is used to connect
// to the default management server. A non-default authority is also
// specified which has resources for resolving the address of the default
// management server.
nodeID := uuid.New().String()
bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: fmt.Appendf(nil, `[{
"server_uri": "xds://%s/%s",
"channel_creds": [{"type": "insecure"}]
}]`, managementServerAuthority, managementServerServiceName),
Node: fmt.Appendf(nil, `{"id": "%s"}`, nodeID),
Authorities: map[string]json.RawMessage{
managementServerAuthority: fmt.Appendf(nil, `{
"xds_servers": [{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]}`, managementServer1.Address),
},
})
if err != nil {
t.Fatalf("Failed to create bootstrap configuration: %v", err)
}
config, err := bootstrap.NewConfigFromContents(bootstrapContents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %v", err)
}
xdsclient.DefaultPool.SetFallbackBootstrapConfig(config)
defer func() { xdsclient.DefaultPool.UnsetBootstrapConfigForTesting() }()
// Update the management server that holds resources for resolving the real
// management server.
managementServerResources := e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(managementServerLDSName, managementServerRDSName)},
Routes: []*v3routepb.RouteConfiguration{
e2e.DefaultRouteConfig(managementServerRDSName, managementServerServiceName, managementServerCDSName),
},
Clusters: []*v3clusterpb.Cluster{
e2e.DefaultCluster(managementServerCDSName, managementServerEDSName, e2e.SecurityLevelNone),
},
Endpoints: []*v3endpointpb.ClusterLoadAssignment{
e2e.DefaultEndpoint(managementServerEDSName, "localhost", []uint32{testutils.ParsePort(t, managementServer2.Address)}),
},
SkipValidation: true,
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if err := managementServer1.Update(ctx, managementServerResources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", managementServerResources, err)
}
// Update the management server that resolves the address of the test stub
// server.
resouces := e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, rdsName)},
Routes: []*v3routepb.RouteConfiguration{
e2e.DefaultRouteConfig(rdsName, serviceName, cdsName),
},
Clusters: []*v3clusterpb.Cluster{
e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone),
},
Endpoints: []*v3endpointpb.ClusterLoadAssignment{
e2e.DefaultEndpoint(edsName, "localhost", []uint32{testutils.ParsePort(t, server.Address)}),
},
SkipValidation: true,
}
if err := managementServer2.Update(ctx, resouces); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resouces, err)
}
cc, err := grpc.NewClient("xds:///"+serviceName, grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("grpc.NewClient() failed: %v", err)
}
defer cc.Close()
client := testgrpc.NewTestServiceClient(cc)
if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
t.Fatalf("rpc EmptyCall() failed: %v", err)
}
}

View File

@@ -0,0 +1,83 @@
/*
*
* Copyright 2020 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient
import (
"fmt"
"sync"
"sync/atomic"
)
type clusterNameAndServiceName struct {
clusterName, edsServiceName string
}
type clusterRequestsCounter struct {
mu sync.Mutex
clusters map[clusterNameAndServiceName]*ClusterRequestsCounter
}
var src = &clusterRequestsCounter{
clusters: make(map[clusterNameAndServiceName]*ClusterRequestsCounter),
}
// ClusterRequestsCounter is used to track the total inflight requests for a
// service with the provided name.
type ClusterRequestsCounter struct {
ClusterName string
EDSServiceName string
numRequests uint32
}
// GetClusterRequestsCounter returns the ClusterRequestsCounter with the
// provided serviceName. If one does not exist, it creates it.
func GetClusterRequestsCounter(clusterName, edsServiceName string) *ClusterRequestsCounter {
src.mu.Lock()
defer src.mu.Unlock()
k := clusterNameAndServiceName{
clusterName: clusterName,
edsServiceName: edsServiceName,
}
c, ok := src.clusters[k]
if !ok {
c = &ClusterRequestsCounter{ClusterName: clusterName}
src.clusters[k] = c
}
return c
}
// StartRequest starts a request for a cluster, incrementing its number of
// requests by 1. Returns an error if the max number of requests is exceeded.
func (c *ClusterRequestsCounter) StartRequest(max uint32) error {
// Note that during race, the limits could be exceeded. This is allowed:
// "Since the implementation is eventually consistent, races between threads
// may allow limits to be potentially exceeded."
// https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/circuit_breaking#arch-overview-circuit-break.
if atomic.LoadUint32(&c.numRequests) >= max {
return fmt.Errorf("max requests %v exceeded on service %v", max, c.ClusterName)
}
atomic.AddUint32(&c.numRequests, 1)
return nil
}
// EndRequest ends a request for a service, decrementing its number of requests
// by 1.
func (c *ClusterRequestsCounter) EndRequest() {
atomic.AddUint32(&c.numRequests, ^uint32(0))
}

View File

@@ -0,0 +1,162 @@
/*
*
* Copyright 2020 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient
import (
"sync"
"sync/atomic"
"testing"
)
const testService = "test-service-name"
type counterTest struct {
name string
maxRequests uint32
numRequests uint32
expectedSuccesses uint32
expectedErrors uint32
}
var tests = []counterTest{
{
name: "does-not-exceed-max-requests",
maxRequests: 1024,
numRequests: 1024,
expectedSuccesses: 1024,
expectedErrors: 0,
},
{
name: "exceeds-max-requests",
maxRequests: 32,
numRequests: 64,
expectedSuccesses: 32,
expectedErrors: 32,
},
}
func resetClusterRequestsCounter() {
src = &clusterRequestsCounter{
clusters: make(map[clusterNameAndServiceName]*ClusterRequestsCounter),
}
}
func testCounter(t *testing.T, test counterTest) {
requestsStarted := make(chan struct{})
requestsSent := sync.WaitGroup{}
requestsSent.Add(int(test.numRequests))
requestsDone := sync.WaitGroup{}
requestsDone.Add(int(test.numRequests))
var lastError atomic.Value
var successes, errors uint32
for i := 0; i < int(test.numRequests); i++ {
go func() {
counter := GetClusterRequestsCounter(test.name, testService)
defer requestsDone.Done()
err := counter.StartRequest(test.maxRequests)
if err == nil {
atomic.AddUint32(&successes, 1)
} else {
atomic.AddUint32(&errors, 1)
lastError.Store(err)
}
requestsSent.Done()
if err == nil {
<-requestsStarted
counter.EndRequest()
}
}()
}
requestsSent.Wait()
close(requestsStarted)
requestsDone.Wait()
loadedError := lastError.Load()
if test.expectedErrors > 0 && loadedError == nil {
t.Error("no error when error expected")
}
if test.expectedErrors == 0 && loadedError != nil {
t.Errorf("error starting request: %v", loadedError.(error))
}
// We allow the limits to be exceeded during races.
//
// But we should never over-limit, so this test fails if there are less
// successes than expected.
if successes < test.expectedSuccesses || errors > test.expectedErrors {
t.Errorf("unexpected number of (successes, errors), expected (%v, %v), encountered (%v, %v)", test.expectedSuccesses, test.expectedErrors, successes, errors)
}
}
func (s) TestRequestsCounter(t *testing.T) {
defer resetClusterRequestsCounter()
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
testCounter(t, test)
})
}
}
func (s) TestGetClusterRequestsCounter(t *testing.T) {
defer resetClusterRequestsCounter()
for _, test := range tests {
counterA := GetClusterRequestsCounter(test.name, testService)
counterB := GetClusterRequestsCounter(test.name, testService)
if counterA != counterB {
t.Errorf("counter %v %v != counter %v %v", counterA, *counterA, counterB, *counterB)
}
}
}
func startRequests(t *testing.T, n uint32, max uint32, counter *ClusterRequestsCounter) {
for i := uint32(0); i < n; i++ {
if err := counter.StartRequest(max); err != nil {
t.Fatalf("error starting initial request: %v", err)
}
}
}
func (s) TestSetMaxRequestsIncreased(t *testing.T) {
defer resetClusterRequestsCounter()
const clusterName string = "set-max-requests-increased"
var initialMax uint32 = 16
counter := GetClusterRequestsCounter(clusterName, testService)
startRequests(t, initialMax, initialMax, counter)
if err := counter.StartRequest(initialMax); err == nil {
t.Fatal("unexpected success on start request after max met")
}
newMax := initialMax + 1
if err := counter.StartRequest(newMax); err != nil {
t.Fatalf("unexpected error on start request after max increased: %v", err)
}
}
func (s) TestSetMaxRequestsDecreased(t *testing.T) {
defer resetClusterRequestsCounter()
const clusterName string = "set-max-requests-decreased"
var initialMax uint32 = 16
counter := GetClusterRequestsCounter(clusterName, testService)
startRequests(t, initialMax-1, initialMax, counter)
newMax := initialMax - 1
if err := counter.StartRequest(newMax); err == nil {
t.Fatalf("unexpected success on start request after max decreased: %v", err)
}
}

View File

@@ -0,0 +1,54 @@
/*
*
* Copyright 2025 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsclient
import (
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/clients/xdsclient"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
)
func supportedResourceTypes(config *bootstrap.Config, gServerCfgMap map[xdsclient.ServerConfig]*bootstrap.ServerConfig) map[string]xdsclient.ResourceType {
return map[string]xdsclient.ResourceType{
version.V3ListenerURL: {
TypeURL: version.V3ListenerURL,
TypeName: xdsresource.ListenerResourceTypeName,
AllResourcesRequiredInSotW: true,
Decoder: xdsresource.NewListenerResourceTypeDecoder(config, gServerCfgMap),
},
version.V3RouteConfigURL: {
TypeURL: version.V3RouteConfigURL,
TypeName: xdsresource.RouteConfigTypeName,
AllResourcesRequiredInSotW: false,
Decoder: xdsresource.NewRouteConfigResourceTypeDecoder(config, gServerCfgMap),
},
version.V3ClusterURL: {
TypeURL: version.V3ClusterURL,
TypeName: xdsresource.ClusterResourceTypeName,
AllResourcesRequiredInSotW: true,
Decoder: xdsresource.NewClusterResourceTypeDecoder(config, gServerCfgMap),
},
version.V3EndpointsURL: {
TypeURL: version.V3EndpointsURL,
TypeName: xdsresource.EndpointsResourceTypeName,
AllResourcesRequiredInSotW: false,
Decoder: xdsresource.NewEndpointsResourceTypeDecoder(config),
},
}
}

View File

@@ -0,0 +1,512 @@
/*
*
* Copyright 2024 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient_test
import (
"context"
"fmt"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/uuid"
"google.golang.org/grpc"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/testutils/xds/e2e"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/xdsclient"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/testing/protocmp"
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
)
// Creates an xDS client with the given bootstrap contents.
func createXDSClient(t *testing.T, bootstrapContents []byte) xdsclient.XDSClient {
t.Helper()
config, err := bootstrap.NewConfigFromContents(bootstrapContents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
}
pool := xdsclient.NewPool(config)
client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: t.Name(),
})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
t.Cleanup(close)
return client
}
// Tests simple ACK and NACK scenarios on the ADS stream:
// 1. When a good response is received, i.e. once that is expected to be ACKed,
// the test verifies that an ACK is sent matching the version and nonce from
// the response.
// 2. When a subsequent bad response is received, i.e. once is expected to be
// NACKed, the test verifies that a NACK is sent matching the previously
// ACKed version and current nonce from the response.
// 3. When a subsequent good response is received, the test verifies that an
// ACK is sent matching the version and nonce from the current response.
func (s) TestADS_ACK_NACK_Simple(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
// Create an xDS management server listening on a local port. Configure the
// request and response handlers to push on channels that are inspected by
// the test goroutine to verify ACK version and nonce.
streamRequestCh := testutils.NewChannel()
streamResponseCh := testutils.NewChannel()
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
streamRequestCh.SendContext(ctx, req)
return nil
},
OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
streamResponseCh.SendContext(ctx, resp)
},
})
// Create a listener resource on the management server.
const listenerName = "listener"
const routeConfigName = "route-config"
nodeID := uuid.New().String()
listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName)
resources := e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{listenerResource},
SkipValidation: true,
}
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatal(err)
}
// Create an xDS client with bootstrap pointing to the above server.
bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
client := createXDSClient(t, bc)
// Register a watch for a listener resource.
lw := newListenerWatcher()
ldsCancel := xdsresource.WatchListener(client, listenerName, lw)
defer ldsCancel()
// Verify that the initial discovery request matches expectation.
r, err := streamRequestCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for the initial discovery request")
}
gotReq := r.(*v3discoverypb.DiscoveryRequest)
wantReq := &v3discoverypb.DiscoveryRequest{
VersionInfo: "",
Node: &v3corepb.Node{
Id: nodeID,
UserAgentName: "gRPC Go",
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
},
ResourceNames: []string{listenerName},
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
ResponseNonce: "",
}
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
}
// Capture the version and nonce from the response.
r, err = streamResponseCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for a discovery response from the server")
}
gotResp := r.(*v3discoverypb.DiscoveryResponse)
// Verify that the ACK contains the appropriate version and nonce.
r, err = streamRequestCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for ACK")
}
gotReq = r.(*v3discoverypb.DiscoveryRequest)
wantReq.VersionInfo = gotResp.GetVersionInfo()
wantReq.ResponseNonce = gotResp.GetNonce()
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
}
// Verify the update received by the watcher.
wantUpdate := listenerUpdateErrTuple{
update: &xdsresource.ListenerUpdate{
APIListener: &xdsresource.HTTPConnectionManagerConfig{
RouteConfigName: routeConfigName,
HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}},
},
},
}
if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
// Update the management server with a listener resource that contains an
// empty HTTP connection manager within the apiListener, which will cause
// the resource to be NACKed.
badListener := proto.Clone(listenerResource).(*v3listenerpb.Listener)
badListener.ApiListener.ApiListener = nil
mgmtServer.Update(ctx, e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{badListener},
SkipValidation: true,
})
r, err = streamResponseCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for a discovery response from the server")
}
gotResp = r.(*v3discoverypb.DiscoveryResponse)
wantNackErr := xdsresource.NewError(xdsresource.ErrorTypeNACKed, "unexpected http connection manager resource type")
if err := verifyListenerUpdate(ctx, lw.updateCh, listenerUpdateErrTuple{err: wantNackErr}); err != nil {
t.Fatal(err)
}
// Verify that the NACK contains the appropriate version, nonce and error.
// We expect the version to not change as this is a NACK.
r, err = streamRequestCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for NACK")
}
gotReq = r.(*v3discoverypb.DiscoveryRequest)
if gotNonce, wantNonce := gotReq.GetResponseNonce(), gotResp.GetNonce(); gotNonce != wantNonce {
t.Errorf("Unexpected nonce in discovery request, got: %v, want: %v", gotNonce, wantNonce)
}
if gotErr := gotReq.GetErrorDetail(); gotErr == nil || !strings.Contains(gotErr.GetMessage(), wantNackErr.Error()) {
t.Fatalf("Unexpected error in discovery request, got: %v, want: %v", gotErr.GetMessage(), wantNackErr)
}
// Update the management server to send a good resource again.
mgmtServer.Update(ctx, e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{listenerResource},
SkipValidation: true,
})
// The envoy-go-control-plane management server keeps resending the same
// resource as long as we keep NACK'ing it. So, we will see the bad resource
// sent to us a few times here, before receiving the good resource.
var lastErr error
for {
if ctx.Err() != nil {
t.Fatalf("Timeout when waiting for an ACK from the xDS client. Last seen error: %v", lastErr)
}
r, err = streamResponseCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for a discovery response from the server")
}
gotResp = r.(*v3discoverypb.DiscoveryResponse)
// Verify that the ACK contains the appropriate version and nonce.
r, err = streamRequestCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for ACK")
}
gotReq = r.(*v3discoverypb.DiscoveryRequest)
wantReq.VersionInfo = gotResp.GetVersionInfo()
wantReq.ResponseNonce = gotResp.GetNonce()
wantReq.ErrorDetail = nil
diff := cmp.Diff(gotReq, wantReq, protocmp.Transform())
if diff == "" {
lastErr = nil
break
}
lastErr = fmt.Errorf("unexpected diff in discovery request, diff (-got, +want):\n%s", diff)
}
// Verify the update received by the watcher.
for ; ctx.Err() == nil; <-time.After(100 * time.Millisecond) {
if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
lastErr = err
continue
}
break
}
if ctx.Err() != nil {
t.Fatalf("Timeout when waiting for listener update. Last seen error: %v", lastErr)
}
}
// Tests the case where the first response is invalid. The test verifies that
// the NACK contains an empty version string.
func (s) TestADS_NACK_InvalidFirstResponse(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
// Create an xDS management server listening on a local port. Configure the
// request and response handlers to push on channels that are inspected by
// the test goroutine to verify ACK version and nonce.
streamRequestCh := testutils.NewChannel()
streamResponseCh := testutils.NewChannel()
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
streamRequestCh.SendContext(ctx, req)
return nil
},
OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
streamResponseCh.SendContext(ctx, resp)
},
})
// Create a listener resource on the management server that is expected to
// be NACKed by the xDS client.
const listenerName = "listener"
const routeConfigName = "route-config"
nodeID := uuid.New().String()
listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName)
listenerResource.ApiListener.ApiListener = nil
resources := e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{listenerResource},
SkipValidation: true,
}
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatal(err)
}
// Create an xDS client with bootstrap pointing to the above server.
bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
client := createXDSClient(t, bc)
// Register a watch for a listener resource.
lw := newListenerWatcher()
ldsCancel := xdsresource.WatchListener(client, listenerName, lw)
defer ldsCancel()
// Verify that the initial discovery request matches expectation.
r, err := streamRequestCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for the initial discovery request")
}
gotReq := r.(*v3discoverypb.DiscoveryRequest)
wantReq := &v3discoverypb.DiscoveryRequest{
VersionInfo: "",
Node: &v3corepb.Node{
Id: nodeID,
UserAgentName: "gRPC Go",
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
},
ResourceNames: []string{listenerName},
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
ResponseNonce: "",
}
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
}
// Capture the version and nonce from the response.
r, err = streamResponseCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for the discovery response from client")
}
gotResp := r.(*v3discoverypb.DiscoveryResponse)
// Verify that the error is propagated to the watcher.
var wantNackErr = xdsresource.NewError(xdsresource.ErrorTypeNACKed, "unexpected http connection manager resource type")
if err := verifyListenerUpdate(ctx, lw.updateCh, listenerUpdateErrTuple{err: wantNackErr}); err != nil {
t.Fatal(err)
}
// NACK should contain the appropriate error, nonce, but empty version.
r, err = streamRequestCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for ACK")
}
gotReq = r.(*v3discoverypb.DiscoveryRequest)
if gotVersion, wantVersion := gotReq.GetVersionInfo(), ""; gotVersion != wantVersion {
t.Errorf("Unexpected version in discovery request, got: %v, want: %v", gotVersion, wantVersion)
}
if gotNonce, wantNonce := gotReq.GetResponseNonce(), gotResp.GetNonce(); gotNonce != wantNonce {
t.Errorf("Unexpected nonce in discovery request, got: %v, want: %v", gotNonce, wantNonce)
}
if gotErr := gotReq.GetErrorDetail(); gotErr == nil || !strings.Contains(gotErr.GetMessage(), wantNackErr.Error()) {
t.Fatalf("Unexpected error in discovery request, got: %v, want: %v", gotErr.GetMessage(), wantNackErr)
}
}
// Tests the scenario where the xDS client is no longer interested in a
// resource. The following sequence of events are tested:
// 1. A resource is requested and a good response is received. The test verifies
// that an ACK is sent for this resource.
// 2. The previously requested resource is no longer requested. The test
// verifies that the connection to the management server is closed.
// 3. The same resource is requested again. The test verifies that a new
// request is sent with an empty version string, which corresponds to the
// first request on a new connection.
func (s) TestADS_ACK_NACK_ResourceIsNotRequestedAnymore(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
// Create an xDS management server listening on a local port. Configure the
// request and response handlers to push on channels that are inspected by
// the test goroutine to verify ACK version and nonce.
streamRequestCh := testutils.NewChannel()
streamResponseCh := testutils.NewChannel()
streamCloseCh := testutils.NewChannel()
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
streamRequestCh.SendContext(ctx, req)
return nil
},
OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
streamResponseCh.SendContext(ctx, resp)
},
OnStreamClosed: func(int64, *v3corepb.Node) {
streamCloseCh.SendContext(ctx, struct{}{})
},
})
// Create a listener resource on the management server.
const listenerName = "listener"
const routeConfigName = "route-config"
nodeID := uuid.New().String()
listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName)
resources := e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{listenerResource},
SkipValidation: true,
}
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatal(err)
}
// Create an xDS client with bootstrap pointing to the above server.
bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
config, err := bootstrap.NewConfigFromContents(bc)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
}
pool := xdsclient.NewPool(config)
client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: t.Name(),
})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
defer close()
// Register a watch for a listener resource.
lw := newListenerWatcher()
ldsCancel := xdsresource.WatchListener(client, listenerName, lw)
defer ldsCancel()
// Verify that the initial discovery request matches expectation.
r, err := streamRequestCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for the initial discovery request")
}
gotReq := r.(*v3discoverypb.DiscoveryRequest)
wantReq := &v3discoverypb.DiscoveryRequest{
VersionInfo: "",
Node: &v3corepb.Node{
Id: nodeID,
UserAgentName: "gRPC Go",
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
},
ResourceNames: []string{listenerName},
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
ResponseNonce: "",
}
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
}
// Capture the version and nonce from the response.
r, err = streamResponseCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for the discovery response from client")
}
gotResp := r.(*v3discoverypb.DiscoveryResponse)
// Verify that the ACK contains the appropriate version and nonce.
r, err = streamRequestCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for ACK")
}
gotReq = r.(*v3discoverypb.DiscoveryRequest)
wantACKReq := proto.Clone(wantReq).(*v3discoverypb.DiscoveryRequest)
wantACKReq.VersionInfo = gotResp.GetVersionInfo()
wantACKReq.ResponseNonce = gotResp.GetNonce()
if diff := cmp.Diff(gotReq, wantACKReq, protocmp.Transform()); diff != "" {
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
}
// Verify the update received by the watcher.
wantUpdate := listenerUpdateErrTuple{
update: &xdsresource.ListenerUpdate{
APIListener: &xdsresource.HTTPConnectionManagerConfig{
RouteConfigName: routeConfigName,
HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}},
},
},
}
if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
// Cancel the watch on the listener resource. This should result in the
// existing connection to be management server getting closed.
ldsCancel()
if _, err := streamCloseCh.Receive(ctx); err != nil {
t.Fatalf("Timeout when expecting existing connection to be closed: %v", err)
}
// There is a race between two events when the last watch on an xdsChannel
// is canceled:
// - an empty discovery request being sent out
// - the ADS stream being closed
// To handle this race, we drain the request channel here so that if an
// empty discovery request was received, it is pulled out of the request
// channel and thereby guaranteeing a clean slate for the next watch
// registered below.
streamRequestCh.Drain()
// Register a watch for the same listener resource.
lw = newListenerWatcher()
ldsCancel = xdsresource.WatchListener(client, listenerName, lw)
defer ldsCancel()
// Verify that the discovery request is identical to the first one sent out
// to the management server.
r, err = streamRequestCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for discovery request")
}
gotReq = r.(*v3discoverypb.DiscoveryRequest)
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
}
// Verify the update received by the watcher.
if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
}

View File

@@ -0,0 +1,196 @@
/*
*
* Copyright 2024 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient_test
import (
"context"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/uuid"
"google.golang.org/grpc"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/testutils/xds/e2e"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/xdsclient"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
"google.golang.org/protobuf/testing/protocmp"
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
)
// Tests that an ADS stream is restarted after a connection failure. Also
// verifies that if there were any watches registered before the connection
// failed, those resources are re-requested after the stream is restarted.
func (s) TestADS_ResourcesAreRequestedAfterStreamRestart(t *testing.T) {
// Create a restartable listener that can simulate a broken ADS stream.
l, err := testutils.LocalTCPListener()
if err != nil {
t.Fatalf("net.Listen() failed: %v", err)
}
lis := testutils.NewRestartableListener(l)
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
// Start an xDS management server that uses a couple of channels to inform
// the test about the request and response messages being exchanged.
streamRequestCh := testutils.NewChannel()
streamResponseCh := testutils.NewChannel()
streamOpened := testutils.NewChannel()
streamClosed := testutils.NewChannel()
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
Listener: lis,
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
t.Logf("Received request for resources: %v of type %s", req.GetResourceNames(), req.GetTypeUrl())
streamRequestCh.SendContext(ctx, req)
return nil
},
OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
streamResponseCh.SendContext(ctx, resp)
},
OnStreamClosed: func(int64, *v3corepb.Node) {
streamClosed.SendContext(ctx, nil)
},
OnStreamOpen: func(context.Context, int64, string) error {
streamOpened.SendContext(ctx, nil)
return nil
},
})
// Create a listener resource on the management server.
const listenerName = "listener"
const routeConfigName = "route-config"
nodeID := uuid.New().String()
resources := e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeConfigName)},
SkipValidation: true,
}
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatal(err)
}
// Create bootstrap configuration pointing to the above management server.
bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
// Create an xDS client with the above bootstrap configuration.
config, err := bootstrap.NewConfigFromContents(bootstrapContents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
}
pool := xdsclient.NewPool(config)
client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: t.Name(),
})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
defer close()
// Register a watch for a listener resource.
lw := newListenerWatcher()
ldsCancel := xdsresource.WatchListener(client, listenerName, lw)
defer ldsCancel()
// Verify that an ADS stream is opened and an LDS request with the above
// resource name is sent.
if _, err = streamOpened.Receive(ctx); err != nil {
t.Fatal("Timeout when waiting for ADS stream to open")
}
// Verify that the initial discovery request matches expectation.
r, err := streamRequestCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for the initial discovery request")
}
gotReq := r.(*v3discoverypb.DiscoveryRequest)
wantReq := &v3discoverypb.DiscoveryRequest{
VersionInfo: "",
Node: &v3corepb.Node{
Id: nodeID,
UserAgentName: "gRPC Go",
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
},
ResourceNames: []string{listenerName},
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
ResponseNonce: "",
}
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
}
// Capture the version and nonce from the response.
r, err = streamResponseCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for a discovery response from the server")
}
gotResp := r.(*v3discoverypb.DiscoveryResponse)
// Verify that the ACK contains the appropriate version and nonce.
r, err = streamRequestCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for ACK")
}
gotReq = r.(*v3discoverypb.DiscoveryRequest)
wantReq.VersionInfo = gotResp.GetVersionInfo()
wantReq.ResponseNonce = gotResp.GetNonce()
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
}
// Verify the update received by the watcher.
wantListenerUpdate := listenerUpdateErrTuple{
update: &xdsresource.ListenerUpdate{
APIListener: &xdsresource.HTTPConnectionManagerConfig{
RouteConfigName: routeConfigName,
HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}},
},
},
}
if err := verifyListenerUpdate(ctx, lw.updateCh, wantListenerUpdate); err != nil {
t.Fatal(err)
}
// Stop the restartable listener and wait for the stream to close.
lis.Stop()
if _, err = streamClosed.Receive(ctx); err != nil {
t.Fatal("Timeout when waiting for ADS stream to close")
}
// Restart the restartable listener and wait for the stream to open.
lis.Restart()
if _, err = streamOpened.Receive(ctx); err != nil {
t.Fatal("Timeout when waiting for ADS stream to open")
}
// Verify that the listener resource is requested again.
r, err = streamRequestCh.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for the initial discovery request")
}
gotReq = r.(*v3discoverypb.DiscoveryRequest)
wantReq.ResponseNonce = ""
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
}
}

View File

@@ -0,0 +1,375 @@
/*
*
* Copyright 2022 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient_test
import (
"context"
"encoding/json"
"fmt"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/google/uuid"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/testutils/xds/e2e"
"google.golang.org/grpc/internal/xds/bootstrap"
xdstestutils "google.golang.org/grpc/internal/xds/testutils"
"google.golang.org/grpc/internal/xds/xdsclient"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
)
const (
testAuthority1 = "test-authority1"
testAuthority2 = "test-authority2"
testAuthority3 = "test-authority3"
)
var (
// These two resources use `testAuthority1`, which contains an empty server
// config in the bootstrap file, and therefore will use the default
// management server.
authorityTestResourceName11 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority1, cdsName+"1", nil)
authorityTestResourceName12 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority1, cdsName+"2", nil)
// This resource uses `testAuthority2`, which contains an empty server
// config in the bootstrap file, and therefore will use the default
// management server.
authorityTestResourceName2 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority2, cdsName+"3", nil)
// This resource uses `testAuthority3`, which contains a non-empty server
// config in the bootstrap file, and therefore will use the non-default
// management server.
authorityTestResourceName3 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority3, cdsName+"3", nil)
)
// setupForAuthorityTests spins up two management servers, one to act as the
// default and the other to act as the non-default. It also generates a
// bootstrap configuration with three authorities (the first two pointing to the
// default and the third one pointing to the non-default).
//
// Returns two listeners used by the default and non-default management servers
// respectively, and the xDS client and its close function.
func setupForAuthorityTests(ctx context.Context, t *testing.T) (*testutils.ListenerWrapper, *testutils.ListenerWrapper, xdsclient.XDSClient, func()) {
// Create listener wrappers which notify on to a channel whenever a new
// connection is accepted. We use this to track the number of transports
// used by the xDS client.
lisDefault := testutils.NewListenerWrapper(t, nil)
lisNonDefault := testutils.NewListenerWrapper(t, nil)
// Start a management server to act as the default authority.
defaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisDefault})
// Start a management server to act as the non-default authority.
nonDefaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisNonDefault})
// Create a bootstrap configuration with two non-default authorities which
// have empty server configs, and therefore end up using the default server
// config, which points to the above management server.
nodeID := uuid.New().String()
bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, defaultAuthorityServer.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
Authorities: map[string]json.RawMessage{
testAuthority1: []byte(`{}`),
testAuthority2: []byte(`{}`),
testAuthority3: []byte(fmt.Sprintf(`{
"xds_servers": [{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]}`, nonDefaultAuthorityServer.Address)),
},
})
if err != nil {
t.Fatalf("Failed to create bootstrap configuration: %v", err)
}
config, err := bootstrap.NewConfigFromContents(bootstrapContents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
}
pool := xdsclient.NewPool(config)
client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: t.Name(),
WatchExpiryTimeout: defaultTestWatchExpiryTimeout,
})
if err != nil {
t.Fatalf("Failed to create an xDS client: %v", err)
}
resources := e2e.UpdateOptions{
NodeID: nodeID,
Clusters: []*v3clusterpb.Cluster{
e2e.DefaultCluster(authorityTestResourceName11, edsName, e2e.SecurityLevelNone),
e2e.DefaultCluster(authorityTestResourceName12, edsName, e2e.SecurityLevelNone),
e2e.DefaultCluster(authorityTestResourceName2, edsName, e2e.SecurityLevelNone),
e2e.DefaultCluster(authorityTestResourceName3, edsName, e2e.SecurityLevelNone),
},
SkipValidation: true,
}
if err := defaultAuthorityServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
return lisDefault, lisNonDefault, client, close
}
// Tests the xdsChannel sharing logic among authorities. The test verifies the
// following scenarios:
// - A watch for a resource name with an authority matching an existing watch
// should not result in a new transport being created.
// - A watch for a resource name with different authority name but same
// authority config as an existing watch should not result in a new transport
// being created.
func (s) TestAuthority_XDSChannelSharing(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
lis, _, client, close := setupForAuthorityTests(ctx, t)
defer close()
// Verify that no connection is established to the management server at this
// point. A transport is created only when a resource (which belongs to that
// authority) is requested.
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
t.Fatal("Unexpected new transport created to management server")
}
// Request the first resource. Verify that a new transport is created.
watcher := noopClusterWatcher{}
cdsCancel1 := xdsresource.WatchCluster(client, authorityTestResourceName11, watcher)
defer cdsCancel1()
if _, err := lis.NewConnCh.Receive(ctx); err != nil {
t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
}
// Request the second resource. Verify that no new transport is created.
cdsCancel2 := xdsresource.WatchCluster(client, authorityTestResourceName12, watcher)
defer cdsCancel2()
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
t.Fatal("Unexpected new transport created to management server")
}
// Request the third resource. Verify that no new transport is created.
cdsCancel3 := xdsresource.WatchCluster(client, authorityTestResourceName2, watcher)
defer cdsCancel3()
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
t.Fatal("Unexpected new transport created to management server")
}
}
// Test the xdsChannel close logic. The test verifies that the xDS client
// closes an xdsChannel immediately after the last watch is canceled.
func (s) TestAuthority_XDSChannelClose(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
lis, _, client, close := setupForAuthorityTests(ctx, t)
defer close()
// Request the first resource. Verify that a new transport is created.
watcher := noopClusterWatcher{}
cdsCancel1 := xdsresource.WatchCluster(client, authorityTestResourceName11, watcher)
val, err := lis.NewConnCh.Receive(ctx)
if err != nil {
t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
}
conn := val.(*testutils.ConnWrapper)
// Request the second resource. Verify that no new transport is created.
cdsCancel2 := xdsresource.WatchCluster(client, authorityTestResourceName12, watcher)
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
t.Fatal("Unexpected new transport created to management server")
}
// Cancel both watches, and verify that the connection to the management
// server is closed.
cdsCancel1()
cdsCancel2()
if _, err := conn.CloseCh.Receive(ctx); err != nil {
t.Fatal("Timeout when waiting for connection to management server to be closed")
}
}
// Tests the scenario where the primary management server is unavailable at
// startup and the xDS client falls back to the secondary. The test verifies
// that the resource watcher is not notifified of the connectivity failure until
// all servers have failed.
func (s) TestAuthority_Fallback(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
// Create primary and secondary management servers with restartable
// listeners.
l, err := testutils.LocalTCPListener()
if err != nil {
t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
}
primaryLis := testutils.NewRestartableListener(l)
primaryMgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis})
l, err = testutils.LocalTCPListener()
if err != nil {
t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
}
secondaryLis := testutils.NewRestartableListener(l)
secondaryMgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: secondaryLis})
// Create bootstrap configuration with the above primary and fallback
// management servers, and an xDS client with that configuration.
nodeID := uuid.New().String()
bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`
[
{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
},
{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}
]`, primaryMgmtServer.Address, secondaryMgmtServer.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
})
if err != nil {
t.Fatalf("Failed to create bootstrap configuration: %v", err)
}
config, err := bootstrap.NewConfigFromContents(bootstrapContents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
}
pool := xdsclient.NewPool(config)
xdsC, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{Name: t.Name()})
if err != nil {
t.Fatalf("Failed to create an xDS client: %v", err)
}
defer close()
const clusterName = "cluster"
const edsPrimaryName = "eds-primary"
const edsSecondaryName = "eds-secondary"
// Create a Cluster resource on the primary.
resources := e2e.UpdateOptions{
NodeID: nodeID,
Clusters: []*v3clusterpb.Cluster{
e2e.DefaultCluster(clusterName, edsPrimaryName, e2e.SecurityLevelNone),
},
SkipValidation: true,
}
if err := primaryMgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update primary management server with resources: %v, err: %v", resources, err)
}
// Create a Cluster resource on the secondary .
resources = e2e.UpdateOptions{
NodeID: nodeID,
Clusters: []*v3clusterpb.Cluster{
e2e.DefaultCluster(clusterName, edsSecondaryName, e2e.SecurityLevelNone),
},
SkipValidation: true,
}
if err := secondaryMgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update primary management server with resources: %v, err: %v", resources, err)
}
// Stop the primary.
primaryLis.Close()
// Register a watch.
watcher := newClusterWatcherV2()
cdsCancel := xdsresource.WatchCluster(xdsC, clusterName, watcher)
defer cdsCancel()
// Ensure that the connectivity error callback is not called.
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
if v, err := watcher.ambientErrCh.Receive(sCtx); err != context.DeadlineExceeded {
t.Fatalf("Error callback on the watcher with error: %v", v.(error))
}
// Ensure that the resource update callback is invoked.
v, err := watcher.updateCh.Receive(ctx)
if err != nil {
t.Fatalf("Error when waiting for a resource update callback: %v", err)
}
gotUpdate := v.(*xdsresource.ClusterUpdate)
wantUpdate := xdsresource.ClusterUpdate{
ClusterName: clusterName,
EDSServiceName: edsSecondaryName,
}
cmpOpts := []cmp.Option{cmpopts.EquateEmpty(), cmpopts.IgnoreFields(xdsresource.ClusterUpdate{}, "Raw", "LBPolicy", "TelemetryLabels")}
if diff := cmp.Diff(wantUpdate, *gotUpdate, cmpOpts...); diff != "" {
t.Fatalf("Diff in the cluster resource update: (-want, got):\n%s", diff)
}
// Stop the secondary.
secondaryLis.Close()
// Ensure that the connectivity error callback is called.
if _, err := watcher.ambientErrCh.Receive(ctx); err != nil {
t.Fatal("Timeout when waiting for error callback on the watcher")
}
}
// TODO: Get rid of the clusterWatcher type in cds_watchers_test.go and use this
// one instead. Also, rename this to clusterWatcher as part of that refactor.
type clusterWatcherV2 struct {
updateCh *testutils.Channel // Messages of type xdsresource.ClusterUpdate
ambientErrCh *testutils.Channel // Messages of type ambient error
resourceErrCh *testutils.Channel // Messages of type resource error
}
func newClusterWatcherV2() *clusterWatcherV2 {
return &clusterWatcherV2{
updateCh: testutils.NewChannel(),
ambientErrCh: testutils.NewChannel(),
resourceErrCh: testutils.NewChannel(),
}
}
func (cw *clusterWatcherV2) ResourceChanged(update *xdsresource.ClusterUpdate, onDone func()) {
cw.updateCh.Send(update)
onDone()
}
func (cw *clusterWatcherV2) AmbientError(err error, onDone func()) {
// When used with a go-control-plane management server that continuously
// resends resources which are NACKed by the xDS client, using a `Replace()`
// here simplifies tests that want access to the most recently received
// error.
cw.ambientErrCh.Replace(err)
onDone()
}
func (cw *clusterWatcherV2) ResourceError(err error, onDone func()) {
// When used with a go-control-plane management server that continuously
// resends resources which are NACKed by the xDS client, using a `Replace()`
// here simplifies tests that want access to the most recently received
// error.
cw.resourceErrCh.Replace(err)
onDone()
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,165 @@
/*
*
* Copyright 2024 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient_test
import (
"context"
"encoding/json"
"fmt"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/uuid"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/internal"
"google.golang.org/grpc/internal/stubserver"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/testutils/xds/e2e"
internalbootstrap "google.golang.org/grpc/internal/xds/bootstrap"
xci "google.golang.org/grpc/internal/xds/xdsclient/internal"
testgrpc "google.golang.org/grpc/interop/grpc_testing"
testpb "google.golang.org/grpc/interop/grpc_testing"
"google.golang.org/grpc/resolver"
"google.golang.org/grpc/xds/bootstrap"
)
// nopDialOption is a no-op grpc.DialOption with a name.
type nopDialOption struct {
grpc.EmptyDialOption
name string
}
// testCredsBundle implements `credentials.Bundle` and `extraDialOptions`.
type testCredsBundle struct {
credentials.Bundle
testDialOptNames []string
}
func (t *testCredsBundle) DialOptions() []grpc.DialOption {
var opts []grpc.DialOption
for _, name := range t.testDialOptNames {
opts = append(opts, &nopDialOption{name: name})
}
return opts
}
type testCredsBuilder struct {
testDialOptNames []string
}
func (t *testCredsBuilder) Build(json.RawMessage) (credentials.Bundle, func(), error) {
return &testCredsBundle{
Bundle: insecure.NewBundle(),
testDialOptNames: t.testDialOptNames,
}, func() {}, nil
}
func (t *testCredsBuilder) Name() string {
return "test_dialer_creds"
}
func (s) TestClientCustomDialOptsFromCredentialsBundle(t *testing.T) {
// Create and register the credentials bundle builder.
credsBuilder := &testCredsBuilder{
testDialOptNames: []string{"opt1", "opt2", "opt3"},
}
bootstrap.RegisterChannelCredentials(credsBuilder)
// Start an xDS management server.
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
// Create bootstrap configuration pointing to the above management server.
nodeID := uuid.New().String()
bc, err := internalbootstrap.NewContentsForTesting(internalbootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[{
"server_uri": %q,
"channel_creds": [{
"type": %q,
"config": {"mgmt_server_address": %q}
}]
}]`, mgmtServer.Address, credsBuilder.Name(), mgmtServer.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
})
if err != nil {
t.Fatalf("Failed to create bootstrap configuration: %v", err)
}
// Create an xDS resolver with the above bootstrap configuration.
var resolverBuilder resolver.Builder
if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
if err != nil {
t.Fatalf("Failed to create xDS resolver for testing: %v", err)
}
}
// Spin up a test backend.
server := stubserver.StartTestService(t, nil)
defer server.Stop()
// Configure client side xDS resources on the management server.
const serviceName = "my-service-client-side-xds"
resources := e2e.DefaultClientResources(e2e.ResourceParams{
DialTarget: serviceName,
NodeID: nodeID,
Host: "localhost",
Port: testutils.ParsePort(t, server.Address),
SecLevel: e2e.SecurityLevelNone,
})
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatal(err)
}
// Intercept a grpc.NewClient call from the xds client to validate DialOptions.
xci.GRPCNewClient = func(target string, opts ...grpc.DialOption) (conn *grpc.ClientConn, err error) {
got := map[string]int{}
for _, opt := range opts {
if mo, ok := opt.(*nopDialOption); ok {
got[mo.name]++
}
}
want := map[string]int{}
for _, name := range credsBuilder.testDialOptNames {
want[name]++
}
if !cmp.Equal(got, want) {
t.Errorf("grpc.NewClient() was called with unexpected DialOptions: got %v, want %v", got, want)
}
return grpc.NewClient(target, opts...)
}
defer func() { xci.GRPCNewClient = grpc.NewClient }()
// Create a ClientConn and make a successful RPC. The insecure transport
// credentials passed into the gRPC.NewClient is the credentials for the
// data plane communication with the test backend.
cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
if err != nil {
t.Fatalf("Failed to dial local test server: %v", err)
}
client := testgrpc.NewTestServiceClient(cc)
if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
t.Fatalf("EmptyCall() failed: %v", err)
}
cc.Close()
}

View File

@@ -0,0 +1,575 @@
/*
*
* Copyright 2022 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient_test
import (
"context"
"encoding/json"
"fmt"
"slices"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/uuid"
"google.golang.org/grpc"
"google.golang.org/grpc/internal/pretty"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/testutils/xds/e2e"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/xdsclient"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
"google.golang.org/protobuf/testing/protocmp"
"google.golang.org/protobuf/types/known/anypb"
v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
"github.com/envoyproxy/go-control-plane/pkg/wellknown"
)
func makeGenericXdsConfig(typeURL, name, version string, status v3adminpb.ClientResourceStatus, config *anypb.Any, failure *v3adminpb.UpdateFailureState) *v3statuspb.ClientConfig_GenericXdsConfig {
return &v3statuspb.ClientConfig_GenericXdsConfig{
TypeUrl: typeURL,
Name: name,
VersionInfo: version,
ClientStatus: status,
XdsConfig: config,
ErrorState: failure,
}
}
func checkResourceDump(ctx context.Context, want *v3statuspb.ClientStatusResponse, pool *xdsclient.Pool) error {
cmpOpts := cmp.Options{
protocmp.Transform(),
protocmp.IgnoreFields((*v3statuspb.ClientConfig_GenericXdsConfig)(nil), "last_updated"),
protocmp.IgnoreFields((*v3adminpb.UpdateFailureState)(nil), "last_update_attempt", "details"),
}
var lastErr error
for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
got := pool.DumpResources()
// Sort the client configs based on the `client_scope` field.
slices.SortFunc(got.GetConfig(), func(a, b *v3statuspb.ClientConfig) int {
return strings.Compare(a.ClientScope, b.ClientScope)
})
// Sort the resource configs based on the type_url and name fields.
for _, cc := range got.GetConfig() {
slices.SortFunc(cc.GetGenericXdsConfigs(), func(a, b *v3statuspb.ClientConfig_GenericXdsConfig) int {
if strings.Compare(a.TypeUrl, b.TypeUrl) == 0 {
return strings.Compare(a.Name, b.Name)
}
return strings.Compare(a.TypeUrl, b.TypeUrl)
})
}
diff := cmp.Diff(want, got, cmpOpts)
if diff == "" {
return nil
}
lastErr = fmt.Errorf("received unexpected resource dump, diff (-want, +got):\n%s, got: %s\n want:%s", diff, pretty.ToJSON(got), pretty.ToJSON(want))
}
return fmt.Errorf("timeout when waiting for resource dump to reach expected state: %v", lastErr)
}
// Tests the scenario where there are multiple xDS clients talking to the same
// management server, and requesting the same set of resources. Verifies that
// under all circumstances, both xDS clients receive the same configuration from
// the server.
func (s) TestDumpResources_ManyToOne(t *testing.T) {
// Initialize the xDS resources to be used in this test.
ldsTargets := []string{"lds.target.good:0000", "lds.target.good:1111"}
rdsTargets := []string{"route-config-0", "route-config-1"}
cdsTargets := []string{"cluster-0", "cluster-1"}
edsTargets := []string{"endpoints-0", "endpoints-1"}
listeners := make([]*v3listenerpb.Listener, len(ldsTargets))
listenerAnys := make([]*anypb.Any, len(ldsTargets))
for i := range ldsTargets {
listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i])
listenerAnys[i] = testutils.MarshalAny(t, listeners[i])
}
routes := make([]*v3routepb.RouteConfiguration, len(rdsTargets))
routeAnys := make([]*anypb.Any, len(rdsTargets))
for i := range rdsTargets {
routes[i] = e2e.DefaultRouteConfig(rdsTargets[i], ldsTargets[i], cdsTargets[i])
routeAnys[i] = testutils.MarshalAny(t, routes[i])
}
clusters := make([]*v3clusterpb.Cluster, len(cdsTargets))
clusterAnys := make([]*anypb.Any, len(cdsTargets))
for i := range cdsTargets {
clusters[i] = e2e.DefaultCluster(cdsTargets[i], edsTargets[i], e2e.SecurityLevelNone)
clusterAnys[i] = testutils.MarshalAny(t, clusters[i])
}
endpoints := make([]*v3endpointpb.ClusterLoadAssignment, len(edsTargets))
endpointAnys := make([]*anypb.Any, len(edsTargets))
ips := []string{"0.0.0.0", "1.1.1.1"}
ports := []uint32{123, 456}
for i := range edsTargets {
endpoints[i] = e2e.DefaultEndpoint(edsTargets[i], ips[i], ports[i:i+1])
endpointAnys[i] = testutils.MarshalAny(t, endpoints[i])
}
// Spin up an xDS management server on a local port.
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
nodeID := uuid.New().String()
bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
config, err := bootstrap.NewConfigFromContents(bc)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
}
pool := xdsclient.NewPool(config)
// Create two xDS clients with the above bootstrap contents.
client1Name := t.Name() + "-1"
client1, close1, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: client1Name,
})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
defer close1()
client2Name := t.Name() + "-2"
client2, close2, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: client2Name,
})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
defer close2()
// Dump resources and expect empty configs.
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
wantNode := &v3corepb.Node{
Id: nodeID,
UserAgentName: "gRPC Go",
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
}
wantResp := &v3statuspb.ClientStatusResponse{
Config: []*v3statuspb.ClientConfig{
{
Node: wantNode,
ClientScope: client1Name,
},
{
Node: wantNode,
ClientScope: client2Name,
},
},
}
if err := checkResourceDump(ctx, wantResp, pool); err != nil {
t.Fatal(err)
}
// Register watches, dump resources and expect configs in requested state.
for _, xdsC := range []xdsclient.XDSClient{client1, client2} {
for _, target := range ldsTargets {
xdsresource.WatchListener(xdsC, target, noopListenerWatcher{})
}
for _, target := range rdsTargets {
xdsresource.WatchRouteConfig(xdsC, target, noopRouteConfigWatcher{})
}
for _, target := range cdsTargets {
xdsresource.WatchCluster(xdsC, target, noopClusterWatcher{})
}
for _, target := range edsTargets {
xdsresource.WatchEndpoints(xdsC, target, noopEndpointsWatcher{})
}
}
wantConfigs := []*v3statuspb.ClientConfig_GenericXdsConfig{
makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
}
wantResp = &v3statuspb.ClientStatusResponse{
Config: []*v3statuspb.ClientConfig{
{
Node: wantNode,
GenericXdsConfigs: wantConfigs,
ClientScope: client1Name,
},
{
Node: wantNode,
GenericXdsConfigs: wantConfigs,
ClientScope: client2Name,
},
},
}
if err := checkResourceDump(ctx, wantResp, pool); err != nil {
t.Fatal(err)
}
// Configure the resources on the management server.
if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
NodeID: nodeID,
Listeners: listeners,
Routes: routes,
Clusters: clusters,
Endpoints: endpoints,
}); err != nil {
t.Fatal(err)
}
// Dump resources and expect ACK configs.
wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{
makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[0], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[1], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[0], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[1], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[0], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, routeAnys[0], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, routeAnys[1], nil),
}
wantResp = &v3statuspb.ClientStatusResponse{
Config: []*v3statuspb.ClientConfig{
{
Node: wantNode,
GenericXdsConfigs: wantConfigs,
ClientScope: client1Name,
},
{
Node: wantNode,
GenericXdsConfigs: wantConfigs,
ClientScope: client2Name,
},
},
}
if err := checkResourceDump(ctx, wantResp, pool); err != nil {
t.Fatal(err)
}
// Update the first resource of each type in the management server to a
// value which is expected to be NACK'ed by the xDS client.
listeners[0] = func() *v3listenerpb.Listener {
hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})},
})
return &v3listenerpb.Listener{
Name: ldsTargets[0],
ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm},
FilterChains: []*v3listenerpb.FilterChain{{
Name: "filter-chain-name",
Filters: []*v3listenerpb.Filter{{
Name: wellknown.HTTPConnectionManager,
ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm},
}},
}},
}
}()
routes[0].VirtualHosts = []*v3routepb.VirtualHost{{Routes: []*v3routepb.Route{{}}}}
clusters[0].ClusterDiscoveryType = &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC}
endpoints[0].Endpoints = []*v3endpointpb.LocalityLbEndpoints{{}}
if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
NodeID: nodeID,
Listeners: listeners,
Routes: routes,
Clusters: clusters,
Endpoints: endpoints,
SkipValidation: true,
}); err != nil {
t.Fatal(err)
}
wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{
makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "1", v3adminpb.ClientResourceStatus_NACKED, clusterAnys[0], &v3adminpb.UpdateFailureState{VersionInfo: "2"}),
makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "2", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[1], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "1", v3adminpb.ClientResourceStatus_NACKED, endpointAnys[0], &v3adminpb.UpdateFailureState{VersionInfo: "2"}),
makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "2", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[1], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_NACKED, listenerAnys[0], &v3adminpb.UpdateFailureState{VersionInfo: "2"}),
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "2", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "1", v3adminpb.ClientResourceStatus_NACKED, routeAnys[0], &v3adminpb.UpdateFailureState{VersionInfo: "2"}),
makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "2", v3adminpb.ClientResourceStatus_ACKED, routeAnys[1], nil),
}
wantResp = &v3statuspb.ClientStatusResponse{
Config: []*v3statuspb.ClientConfig{
{
Node: wantNode,
GenericXdsConfigs: wantConfigs,
ClientScope: client1Name,
},
{
Node: wantNode,
GenericXdsConfigs: wantConfigs,
ClientScope: client2Name,
},
},
}
if err := checkResourceDump(ctx, wantResp, pool); err != nil {
t.Fatal(err)
}
}
// Tests the scenario where there are multiple xDS client talking to different
// management server, and requesting different set of resources.
func (s) TestDumpResources_ManyToMany(t *testing.T) {
// Initialize the xDS resources to be used in this test:
// - The first xDS client watches old style resource names, and thereby
// requests these resources from the top-level xDS server.
// - The second xDS client watches new style resource names with a non-empty
// authority, and thereby requests these resources from the server
// configuration for that authority.
authority := strings.Join(strings.Split(t.Name(), "/"), "")
ldsTargets := []string{
"lds.target.good:0000",
fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/lds.targer.good:1111", authority),
}
rdsTargets := []string{
"route-config-0",
fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/route-config-1", authority),
}
cdsTargets := []string{
"cluster-0",
fmt.Sprintf("xdstp://%s/envoy.config.cluster.v3.Cluster/cluster-1", authority),
}
edsTargets := []string{
"endpoints-0",
fmt.Sprintf("xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/endpoints-1", authority),
}
listeners := make([]*v3listenerpb.Listener, len(ldsTargets))
listenerAnys := make([]*anypb.Any, len(ldsTargets))
for i := range ldsTargets {
listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i])
listenerAnys[i] = testutils.MarshalAny(t, listeners[i])
}
routes := make([]*v3routepb.RouteConfiguration, len(rdsTargets))
routeAnys := make([]*anypb.Any, len(rdsTargets))
for i := range rdsTargets {
routes[i] = e2e.DefaultRouteConfig(rdsTargets[i], ldsTargets[i], cdsTargets[i])
routeAnys[i] = testutils.MarshalAny(t, routes[i])
}
clusters := make([]*v3clusterpb.Cluster, len(cdsTargets))
clusterAnys := make([]*anypb.Any, len(cdsTargets))
for i := range cdsTargets {
clusters[i] = e2e.DefaultCluster(cdsTargets[i], edsTargets[i], e2e.SecurityLevelNone)
clusterAnys[i] = testutils.MarshalAny(t, clusters[i])
}
endpoints := make([]*v3endpointpb.ClusterLoadAssignment, len(edsTargets))
endpointAnys := make([]*anypb.Any, len(edsTargets))
ips := []string{"0.0.0.0", "1.1.1.1"}
ports := []uint32{123, 456}
for i := range edsTargets {
endpoints[i] = e2e.DefaultEndpoint(edsTargets[i], ips[i], ports[i:i+1])
endpointAnys[i] = testutils.MarshalAny(t, endpoints[i])
}
// Start two management servers.
mgmtServer1 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
mgmtServer2 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
// The first of the above management servers will be the top-level xDS
// server in the bootstrap configuration, and the second will be the xDS
// server corresponding to the test authority.
nodeID := uuid.New().String()
bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, mgmtServer1.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
Authorities: map[string]json.RawMessage{
authority: []byte(fmt.Sprintf(`{
"xds_servers": [{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]}`, mgmtServer2.Address)),
},
})
if err != nil {
t.Fatalf("Failed to create bootstrap configuration: %v", err)
}
config, err := bootstrap.NewConfigFromContents(bc)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
}
pool := xdsclient.NewPool(config)
// Create two xDS clients with the above bootstrap contents.
client1Name := t.Name() + "-1"
client1, close1, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: client1Name,
})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
defer close1()
client2Name := t.Name() + "-2"
client2, close2, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: client2Name,
})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
defer close2()
// Check the resource dump before configuring resources on the management server.
// Dump resources and expect empty configs.
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
wantNode := &v3corepb.Node{
Id: nodeID,
UserAgentName: "gRPC Go",
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
}
wantResp := &v3statuspb.ClientStatusResponse{
Config: []*v3statuspb.ClientConfig{
{
Node: wantNode,
ClientScope: client1Name,
},
{
Node: wantNode,
ClientScope: client2Name,
},
},
}
if err := checkResourceDump(ctx, wantResp, pool); err != nil {
t.Fatal(err)
}
// Register watches, the first xDS client watches old style resource names,
// while the second xDS client watches new style resource names.
xdsresource.WatchListener(client1, ldsTargets[0], noopListenerWatcher{})
xdsresource.WatchRouteConfig(client1, rdsTargets[0], noopRouteConfigWatcher{})
xdsresource.WatchCluster(client1, cdsTargets[0], noopClusterWatcher{})
xdsresource.WatchEndpoints(client1, edsTargets[0], noopEndpointsWatcher{})
xdsresource.WatchListener(client2, ldsTargets[1], noopListenerWatcher{})
xdsresource.WatchRouteConfig(client2, rdsTargets[1], noopRouteConfigWatcher{})
xdsresource.WatchCluster(client2, cdsTargets[1], noopClusterWatcher{})
xdsresource.WatchEndpoints(client2, edsTargets[1], noopEndpointsWatcher{})
// Check the resource dump. Both clients should have all resources in
// REQUESTED state.
wantConfigs1 := []*v3statuspb.ClientConfig_GenericXdsConfig{
makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
}
wantConfigs2 := []*v3statuspb.ClientConfig_GenericXdsConfig{
makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
}
wantResp = &v3statuspb.ClientStatusResponse{
Config: []*v3statuspb.ClientConfig{
{
Node: wantNode,
GenericXdsConfigs: wantConfigs1,
ClientScope: client1Name,
},
{
Node: wantNode,
GenericXdsConfigs: wantConfigs2,
ClientScope: client2Name,
},
},
}
if err := checkResourceDump(ctx, wantResp, pool); err != nil {
t.Fatal(err)
}
// Configure resources on the first management server.
if err := mgmtServer1.Update(ctx, e2e.UpdateOptions{
NodeID: nodeID,
Listeners: listeners[:1],
Routes: routes[:1],
Clusters: clusters[:1],
Endpoints: endpoints[:1],
}); err != nil {
t.Fatal(err)
}
// Check the resource dump. One client should have resources in ACKED state,
// while the other should still have resources in REQUESTED state.
wantConfigs1 = []*v3statuspb.ClientConfig_GenericXdsConfig{
makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[0], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[0], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[0], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, routeAnys[0], nil),
}
wantResp = &v3statuspb.ClientStatusResponse{
Config: []*v3statuspb.ClientConfig{
{
Node: wantNode,
GenericXdsConfigs: wantConfigs1,
ClientScope: client1Name,
},
{
Node: wantNode,
GenericXdsConfigs: wantConfigs2,
ClientScope: client2Name,
},
},
}
if err := checkResourceDump(ctx, wantResp, pool); err != nil {
t.Fatal(err)
}
// Configure resources on the second management server.
if err := mgmtServer2.Update(ctx, e2e.UpdateOptions{
NodeID: nodeID,
Listeners: listeners[1:],
Routes: routes[1:],
Clusters: clusters[1:],
Endpoints: endpoints[1:],
}); err != nil {
t.Fatal(err)
}
// Check the resource dump. Both clients should have appropriate resources
// in REQUESTED state.
wantConfigs2 = []*v3statuspb.ClientConfig_GenericXdsConfig{
makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[1], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[1], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil),
makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, routeAnys[1], nil),
}
wantResp = &v3statuspb.ClientStatusResponse{
Config: []*v3statuspb.ClientConfig{
{
Node: wantNode,
GenericXdsConfigs: wantConfigs1,
ClientScope: client1Name,
},
{
Node: wantNode,
GenericXdsConfigs: wantConfigs2,
ClientScope: client2Name,
},
},
}
if err := checkResourceDump(ctx, wantResp, pool); err != nil {
t.Fatal(err)
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,931 @@
/*
*
* Copyright 2024 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient_fallback_test
import (
"context"
"fmt"
"sync/atomic"
"testing"
"time"
"github.com/google/uuid"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/internal"
"google.golang.org/grpc/internal/grpctest"
"google.golang.org/grpc/internal/stubserver"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/testutils/xds/e2e"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/xdsclient"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/resolver"
"google.golang.org/grpc/status"
v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
testgrpc "google.golang.org/grpc/interop/grpc_testing"
testpb "google.golang.org/grpc/interop/grpc_testing"
_ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter.
_ "google.golang.org/grpc/xds" // To ensure internal.NewXDSResolverWithConfigForTesting is set.
)
type s struct {
grpctest.Tester
}
func Test(t *testing.T) {
grpctest.RunSubTests(t, s{})
}
// Give the fallback tests additional time to complete because they need to
// first identify failed connections before establishing new ones.
const defaultFallbackTestTimeout = 20 * time.Second
const defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen.
func waitForRPCsToReachBackend(ctx context.Context, client testgrpc.TestServiceClient, backend string) error {
var lastErr error
for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
var peer peer.Peer
if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil {
lastErr = err
continue
}
// Veirfy the peer when the RPC succeeds.
if peer.Addr.String() == backend {
break
}
}
if ctx.Err() != nil {
return fmt.Errorf("timeout when waiting for RPCs to reach expected backend. Last error: %v", lastErr)
}
return nil
}
// Tests fallback on startup where the xDS client is unable to establish a
// connection to the primary server. The test verifies that the xDS client falls
// back to the secondary server, and when the primary comes back up, it reverts
// to it. The test also verifies that when all requested resources are cached
// from the primary, fallback is not triggered when the connection goes down.
func (s) TestFallback_OnStartup(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout)
defer cancel()
// Create two listeners for the two management servers. The test can
// start/stop these listeners and can also get notified when the listener
// receives a connection request.
primaryWrappedLis := testutils.NewListenerWrapper(t, nil)
primaryLis := testutils.NewRestartableListener(primaryWrappedLis)
fallbackWrappedLis := testutils.NewListenerWrapper(t, nil)
fallbackLis := testutils.NewRestartableListener(fallbackWrappedLis)
// Start two management servers, primary and fallback, with the above
// listeners.
primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis})
fallbackManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: fallbackLis})
// Start two test service backends.
backend1 := stubserver.StartTestService(t, nil)
defer backend1.Stop()
backend2 := stubserver.StartTestService(t, nil)
defer backend2.Stop()
// Configure xDS resource on the primary management server, with a cluster
// resource that contains an endpoint for backend1.
nodeID := uuid.New().String()
const serviceName = "my-service-fallback-xds"
resources := e2e.DefaultClientResources(e2e.ResourceParams{
DialTarget: serviceName,
NodeID: nodeID,
Host: "localhost",
Port: testutils.ParsePort(t, backend1.Address),
SecLevel: e2e.SecurityLevelNone,
})
if err := primaryManagementServer.Update(ctx, resources); err != nil {
t.Fatal(err)
}
// Configure xDS resource on the secondary management server, with a cluster
// resource that contains an endpoint for backend2. Only the listener
// resource has the same name on both servers.
fallbackRouteConfigName := "fallback-route-" + serviceName
fallbackClusterName := "fallback-cluster-" + serviceName
fallbackEndpointsName := "fallback-endpoints-" + serviceName
resources = e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, fallbackRouteConfigName)},
Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(fallbackRouteConfigName, serviceName, fallbackClusterName)},
Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(fallbackClusterName, fallbackEndpointsName, e2e.SecurityLevelNone)},
Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(fallbackEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend2.Address)})},
}
if err := fallbackManagementServer.Update(ctx, resources); err != nil {
t.Fatal(err)
}
// Shut both management servers down before starting the gRPC client to
// trigger fallback on startup.
primaryLis.Stop()
fallbackLis.Stop()
// Generate bootstrap configuration with the above two servers.
bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[
{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
},
{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, primaryManagementServer.Address, fallbackManagementServer.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
})
if err != nil {
t.Fatalf("Failed to create bootstrap file: %v", err)
}
// Create an xDS client with the above bootstrap configuration.
config, err := bootstrap.NewConfigFromContents(bootstrapContents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
}
pool := xdsclient.NewPool(config)
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
// Get the xDS resolver to use the above xDS client.
resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error))
resolver, err := resolverBuilder(pool)
if err != nil {
t.Fatalf("Failed to create xDS resolver for testing: %v", err)
}
// Start a gRPC client that uses the above xDS resolver.
cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
if err != nil {
t.Fatalf("Failed to create gRPC client: %v", err)
}
defer cc.Close()
cc.Connect()
// Ensure that a connection is attempted to the primary.
if _, err := primaryWrappedLis.NewConnCh.Receive(ctx); err != nil {
t.Fatalf("Failure when waiting for a connection to be opened to the primary management server: %v", err)
}
// Ensure that a connection is attempted to the fallback.
if _, err := fallbackWrappedLis.NewConnCh.Receive(ctx); err != nil {
t.Fatalf("Failure when waiting for a connection to be opened to the primary management server: %v", err)
}
// Make an RPC with a shortish deadline and expect it to fail.
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
client := testgrpc.NewTestServiceClient(cc)
if _, err := client.EmptyCall(sCtx, &testpb.Empty{}, grpc.WaitForReady(true)); err == nil || status.Code(err) != codes.DeadlineExceeded {
t.Fatalf("EmptyCall() = %v, want DeadlineExceeded", err)
}
// Start the fallback server. Ensure that an RPC can succeed, and that it
// reaches backend2.
fallbackLis.Restart()
if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil {
t.Fatal(err)
}
// Start the primary server. It can take a while before the xDS client
// notices this, since the ADS stream implementation uses a backoff before
// retrying the stream.
primaryLis.Restart()
// Wait for the connection to the secondary to be closed and ensure that an
// RPC can succeed, and that it reaches backend1.
c, err := fallbackWrappedLis.NewConnCh.Receive(ctx)
if err != nil {
t.Fatalf("Failure when retrieving the most recent connection to the fallback management server: %v", err)
}
conn := c.(*testutils.ConnWrapper)
if _, err := conn.CloseCh.Receive(ctx); err != nil {
t.Fatalf("Connection to fallback server not closed once primary becomes ready: %v", err)
}
if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
t.Fatal(err)
}
// Stop the primary servers. Since all xDS resources were received from the
// primary (and RPCs were succeeding to the clusters returned by the
// primary), we will not trigger fallback.
primaryLis.Stop()
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
if _, err := fallbackWrappedLis.NewConnCh.Receive(sCtx); err == nil {
t.Fatalf("Fallback attempted when not expected to. There are no uncached resources from the primary server at this point.")
}
// Ensure that RPCs still succeed, and that they use the configuration
// received from the primary.
if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
t.Fatal(err)
}
}
// Tests fallback when the primary management server fails during an update.
func (s) TestFallback_MidUpdate(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout)
defer cancel()
// Create two listeners for the two management servers. The test can
// start/stop these listeners and can also get notified when the listener
// receives a connection request.
primaryWrappedLis := testutils.NewListenerWrapper(t, nil)
primaryLis := testutils.NewRestartableListener(primaryWrappedLis)
fallbackWrappedLis := testutils.NewListenerWrapper(t, nil)
fallbackLis := testutils.NewRestartableListener(fallbackWrappedLis)
// This boolean helps with triggering fallback mid update. When this boolean
// is set and the below defined cluster resource is requested, the primary
// management server shuts down the connection, forcing the client to
// fallback to the secondary server.
var closeConnOnMidUpdateClusterResource atomic.Bool
const (
serviceName = "my-service-fallback-xds"
routeConfigName = "route-" + serviceName
clusterName = "cluster-" + serviceName
endpointsName = "endpoints-" + serviceName
midUpdateRouteConfigName = "mid-update-route-" + serviceName
midUpdateClusterName = "mid-update-cluster-" + serviceName
midUpdateEndpointsName = "mid-update-endpoints-" + serviceName
)
// Start two management servers, primary and fallback, with the above
// listeners.
primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
Listener: primaryLis,
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
if closeConnOnMidUpdateClusterResource.Load() == false {
return nil
}
if req.GetTypeUrl() != version.V3ClusterURL {
return nil
}
for _, name := range req.GetResourceNames() {
if name == midUpdateClusterName {
primaryLis.Stop()
return fmt.Errorf("closing ADS stream because %q resource was requested", midUpdateClusterName)
}
}
return nil
},
AllowResourceSubset: true,
})
fallbackManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: fallbackLis})
// Start three test service backends.
backend1 := stubserver.StartTestService(t, nil)
defer backend1.Stop()
backend2 := stubserver.StartTestService(t, nil)
defer backend2.Stop()
backend3 := stubserver.StartTestService(t, nil)
defer backend3.Stop()
// Configure xDS resource on the primary management server, with a cluster
// resource that contains an endpoint for backend1.
nodeID := uuid.New().String()
primaryResources := e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelNone)},
Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, "localhost", []uint32{testutils.ParsePort(t, backend1.Address)})},
}
if err := primaryManagementServer.Update(ctx, primaryResources); err != nil {
t.Fatal(err)
}
// Configure xDS resource on the secondary management server, with a cluster
// resource that contains an endpoint for backend2. Only the listener
// resource has the same name on both servers.
const (
fallbackRouteConfigName = "fallback-route-" + serviceName
fallbackClusterName = "fallback-cluster-" + serviceName
fallbackEndpointsName = "fallback-endpoints-" + serviceName
)
fallbackResources := e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, fallbackRouteConfigName)},
Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(fallbackRouteConfigName, serviceName, fallbackClusterName)},
Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(fallbackClusterName, fallbackEndpointsName, e2e.SecurityLevelNone)},
Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(fallbackEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend2.Address)})},
}
if err := fallbackManagementServer.Update(ctx, fallbackResources); err != nil {
t.Fatal(err)
}
// Generate bootstrap configuration with the above two servers.
bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[
{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
},
{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, primaryManagementServer.Address, fallbackManagementServer.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
})
if err != nil {
t.Fatalf("Failed to create bootstrap file: %v", err)
}
// Create an xDS client with the above bootstrap configuration.
config, err := bootstrap.NewConfigFromContents(bootstrapContents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
}
pool := xdsclient.NewPool(config)
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
// Get the xDS resolver to use the above xDS client.
resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error))
resolver, err := resolverBuilder(pool)
if err != nil {
t.Fatalf("Failed to create xDS resolver for testing: %v", err)
}
// Start a gRPC client that uses the above xDS resolver.
cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
if err != nil {
t.Fatalf("Failed to create gRPC client: %v", err)
}
defer cc.Close()
cc.Connect()
// Ensure that RPCs reach the cluster specified by the primary server and
// that no connection is attempted to the fallback server.
client := testgrpc.NewTestServiceClient(cc)
if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
t.Fatal(err)
}
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
if _, err := fallbackWrappedLis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
t.Fatalf("Connection attempt made to fallback server when none expected: %v", err)
}
// Instruct the primary server to close the connection if below defined
// cluster resource is requested.
closeConnOnMidUpdateClusterResource.Store(true)
// Update the listener resource on the primary server to point to a new
// route configuration that points to a new cluster that points to a new
// endpoints resource that contains backend3.
primaryResources = e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, midUpdateRouteConfigName)},
Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(midUpdateRouteConfigName, serviceName, midUpdateClusterName)},
Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(midUpdateClusterName, midUpdateEndpointsName, e2e.SecurityLevelNone)},
Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(midUpdateEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend3.Address)})},
}
if err := primaryManagementServer.Update(ctx, primaryResources); err != nil {
t.Fatal(err)
}
// Ensure that a connection is attempted to the fallback (because both
// conditions mentioned for fallback in A71 are satisfied: connectivity
// failure and a watcher for an uncached resource), and that RPCs are
// routed to the cluster returned by the fallback server.
c, err := fallbackWrappedLis.NewConnCh.Receive(ctx)
if err != nil {
t.Fatalf("Failure when waiting for a connection to be opened to the fallback management server: %v", err)
}
fallbackConn := c.(*testutils.ConnWrapper)
if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil {
t.Fatal(err)
}
// Set the primary management server to not close the connection anymore if
// the mid-update cluster resource is requested, and get it to start serving
// again.
closeConnOnMidUpdateClusterResource.Store(false)
primaryLis.Restart()
// A new snapshot, with the same resources, is pushed to the management
// server to get it to respond for already requested resource names.
if err := primaryManagementServer.Update(ctx, primaryResources); err != nil {
t.Fatal(err)
}
// Ensure that RPCs reach the backend pointed to by the new cluster.
if err := waitForRPCsToReachBackend(ctx, client, backend3.Address); err != nil {
t.Fatal(err)
}
// Wait for the connection to the secondary to be closed since we have
// reverted back to the primary.
if _, err := fallbackConn.CloseCh.Receive(ctx); err != nil {
t.Fatalf("Connection to fallback server not closed once primary becomes ready: %v", err)
}
}
// Tests fallback when the primary management server fails during startup.
func (s) TestFallback_MidStartup(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout)
defer cancel()
// Create two listeners for the two management servers. The test can
// start/stop these listeners and can also get notified when the listener
// receives a connection request.
primaryWrappedLis := testutils.NewListenerWrapper(t, nil)
primaryLis := testutils.NewRestartableListener(primaryWrappedLis)
fallbackWrappedLis := testutils.NewListenerWrapper(t, nil)
fallbackLis := testutils.NewRestartableListener(fallbackWrappedLis)
// This boolean helps with triggering fallback during startup. When this
// boolean is set and a cluster resource is requested, the primary
// management server shuts down the connection, forcing the client to
// fallback to the secondary server.
var closeConnOnClusterResource atomic.Bool
closeConnOnClusterResource.Store(true)
// Start two management servers, primary and fallback, with the above
// listeners.
primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
Listener: primaryLis,
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
if closeConnOnClusterResource.Load() == false {
return nil
}
if req.GetTypeUrl() != version.V3ClusterURL {
return nil
}
primaryLis.Stop()
return fmt.Errorf("closing ADS stream because cluster resource was requested")
},
AllowResourceSubset: true,
})
fallbackManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: fallbackLis})
// Start two test service backends.
backend1 := stubserver.StartTestService(t, nil)
defer backend1.Stop()
backend2 := stubserver.StartTestService(t, nil)
defer backend2.Stop()
// Configure xDS resource on the primary management server, with a cluster
// resource that contains an endpoint for backend1.
nodeID := uuid.New().String()
const serviceName = "my-service-fallback-xds"
primaryResources := e2e.DefaultClientResources(e2e.ResourceParams{
DialTarget: serviceName,
NodeID: nodeID,
Host: "localhost",
Port: testutils.ParsePort(t, backend1.Address),
SecLevel: e2e.SecurityLevelNone,
})
if err := primaryManagementServer.Update(ctx, primaryResources); err != nil {
t.Fatal(err)
}
// Configure xDS resource on the secondary management server, with a cluster
// resource that contains an endpoint for backend2. Only the listener
// resource has the same name on both servers.
fallbackRouteConfigName := "fallback-route-" + serviceName
fallbackClusterName := "fallback-cluster-" + serviceName
fallbackEndpointsName := "fallback-endpoints-" + serviceName
fallbackResources := e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, fallbackRouteConfigName)},
Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(fallbackRouteConfigName, serviceName, fallbackClusterName)},
Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(fallbackClusterName, fallbackEndpointsName, e2e.SecurityLevelNone)},
Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(fallbackEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend2.Address)})},
}
if err := fallbackManagementServer.Update(ctx, fallbackResources); err != nil {
t.Fatal(err)
}
// Generate bootstrap configuration with the above two servers.
bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[
{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
},
{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, primaryManagementServer.Address, fallbackManagementServer.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
})
if err != nil {
t.Fatalf("Failed to create bootstrap file: %v", err)
}
// Create an xDS client with the above bootstrap configuration.
config, err := bootstrap.NewConfigFromContents(bootstrapContents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
}
pool := xdsclient.NewPool(config)
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
// Get the xDS resolver to use the above xDS client.
resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error))
resolver, err := resolverBuilder(pool)
if err != nil {
t.Fatalf("Failed to create xDS resolver for testing: %v", err)
}
// Start a gRPC client that uses the above xDS resolver.
cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
if err != nil {
t.Fatalf("Failed to create gRPC client: %v", err)
}
defer cc.Close()
cc.Connect()
// Ensure that a connection is attempted to the primary.
if _, err := primaryWrappedLis.NewConnCh.Receive(ctx); err != nil {
t.Fatalf("Failure when waiting for a connection to be opened to the primary management server: %v", err)
}
// Ensure that a connection is attempted to the fallback.
c, err := fallbackWrappedLis.NewConnCh.Receive(ctx)
if err != nil {
t.Fatalf("Failure when waiting for a connection to be opened to the secondary management server: %v", err)
}
fallbackConn := c.(*testutils.ConnWrapper)
// Ensure that RPCs are routed to the cluster returned by the fallback
// management server.
client := testgrpc.NewTestServiceClient(cc)
if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil {
t.Fatal(err)
}
// Get the primary management server to no longer close the connection when
// the cluster resource is requested.
closeConnOnClusterResource.Store(false)
primaryLis.Restart()
// A new snapshot, with the same resources, is pushed to the management
// server to get it to respond for already requested resource names.
if err := primaryManagementServer.Update(ctx, primaryResources); err != nil {
t.Fatal(err)
}
// Ensure that RPCs are routed to the cluster returned by the primary
// management server.
if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
t.Fatal(err)
}
// Wait for the connection to the secondary to be closed since we have
// reverted back to the primary.
if _, err := fallbackConn.CloseCh.Receive(ctx); err != nil {
t.Fatalf("Connection to fallback server not closed once primary becomes ready: %v", err)
}
}
// Tests that RPCs succeed at startup when the primary management server is
// down, but the secondary is available.
func (s) TestFallback_OnStartup_RPCSuccess(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout)
defer cancel()
// Create two listeners for the two management servers. The test can
// start/stop these listeners.
l, err := testutils.LocalTCPListener()
if err != nil {
t.Fatalf("Failed to create listener: %v", err)
}
primaryLis := testutils.NewRestartableListener(l)
l, err = testutils.LocalTCPListener()
if err != nil {
t.Fatalf("Failed to create listener: %v", err)
}
fallbackLis := testutils.NewRestartableListener(l)
// Start two management servers, primary and fallback, with the above
// listeners.
primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis})
fallbackManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: fallbackLis})
// Start two test service backends.
backend1 := stubserver.StartTestService(t, nil)
defer backend1.Stop()
backend2 := stubserver.StartTestService(t, nil)
defer backend2.Stop()
// Configure xDS resource on the primary management server, with a cluster
// resource that contains an endpoint for backend1.
nodeID := uuid.New().String()
const serviceName = "my-service-fallback-xds"
resources := e2e.DefaultClientResources(e2e.ResourceParams{
DialTarget: serviceName,
NodeID: nodeID,
Host: "localhost",
Port: testutils.ParsePort(t, backend1.Address),
SecLevel: e2e.SecurityLevelNone,
})
if err := primaryManagementServer.Update(ctx, resources); err != nil {
t.Fatal(err)
}
// Configure xDS resource on the secondary management server, with a cluster
// resource that contains an endpoint for backend2. Only the listener
// resource has the same name on both servers.
fallbackRouteConfigName := "fallback-route-" + serviceName
fallbackClusterName := "fallback-cluster-" + serviceName
fallbackEndpointsName := "fallback-endpoints-" + serviceName
resources = e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, fallbackRouteConfigName)},
Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(fallbackRouteConfigName, serviceName, fallbackClusterName)},
Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(fallbackClusterName, fallbackEndpointsName, e2e.SecurityLevelNone)},
Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(fallbackEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend2.Address)})},
}
if err := fallbackManagementServer.Update(ctx, resources); err != nil {
t.Fatal(err)
}
// Shutdown the primary management server before starting the gRPC client to
// trigger fallback on startup.
primaryLis.Stop()
// Generate bootstrap configuration with the above two servers.
bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[
{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
},
{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, primaryManagementServer.Address, fallbackManagementServer.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
})
if err != nil {
t.Fatalf("Failed to create bootstrap file: %v", err)
}
// Create an xDS client with the above bootstrap configuration.
config, err := bootstrap.NewConfigFromContents(bootstrapContents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
}
pool := xdsclient.NewPool(config)
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
// Get the xDS resolver to use the above xDS client.
resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error))
resolver, err := resolverBuilder(pool)
if err != nil {
t.Fatalf("Failed to create xDS resolver for testing: %v", err)
}
// Start a gRPC client that uses the above xDS resolver.
cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
if err != nil {
t.Fatalf("Failed to create gRPC client: %v", err)
}
defer cc.Close()
// Make an RPC (without the `wait_for_ready` call option) and expect it to
// succeed since the fallback management server is up and running.
client := testgrpc.NewTestServiceClient(cc)
var peer peer.Peer
if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil {
t.Fatalf("EmptyCall() failed: %v", err)
}
if got, want := peer.Addr.String(), backend2.Address; got != want {
t.Fatalf("Unexpected peer address: got %q, want %q", got, want)
}
// Start the primary server. It can take a while before the xDS client
// notices this, since the ADS stream implementation uses a backoff before
// retrying the stream.
primaryLis.Restart()
if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
t.Fatal(err)
}
}
// Test verifies that when the primary management server is unavailable, the
// system attempts to connect to the first fallback server, and if that is also
// down, to the second fallback server. It also ensures that the system switches
// back to the first fallback server once it becomes available again, and
// eventually returns to the primary server when it comes back online, closing
// connections to the fallback servers accordingly.
func (s) TestFallback_ThreeServerPromotion(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout)
defer cancel()
// Create three listener wrappers for three management servers.
primaryWrappedLis := testutils.NewListenerWrapper(t, nil)
primaryLis := testutils.NewRestartableListener(primaryWrappedLis)
secondaryWrappedLis := testutils.NewListenerWrapper(t, nil)
secondaryLis := testutils.NewRestartableListener(secondaryWrappedLis)
tertiaryWrappedLis := testutils.NewListenerWrapper(t, nil)
tertiaryLis := testutils.NewRestartableListener(tertiaryWrappedLis)
// Start the three management servers.
primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis})
secondaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: secondaryLis})
tertiaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: tertiaryLis})
// Start three test service backends.
backend1 := stubserver.StartTestService(t, nil)
backend1Port := testutils.ParsePort(t, backend1.Address)
defer backend1.Stop()
backend2 := stubserver.StartTestService(t, nil)
backend2Port := testutils.ParsePort(t, backend2.Address)
defer backend2.Stop()
backend3 := stubserver.StartTestService(t, nil)
backend3Port := testutils.ParsePort(t, backend3.Address)
defer backend3.Stop()
nodeID := uuid.New().String()
const (
serviceName = "my-service-fallback-xds"
primaryRouteConfigName = "primary-route-" + serviceName
secondaryRouteConfigName = "secondary-route-" + serviceName
tertiaryRouteConfigName = "tertiary-route-" + serviceName
primaryClusterName = "primary-cluster-" + serviceName
secondaryClusterName = "secondary-cluster-" + serviceName
tertiaryClusterName = "tertiary-cluster-" + serviceName
primaryEndpointsName = "primary-endpoints-" + serviceName
secondaryEndpointsName = "secondary-endpoints-" + serviceName
tertiaryEndpointsName = "tertiary-endpoints-" + serviceName
)
// Configure partial resources on the primary and secondary
// management servers.
primaryManagementServer.Update(ctx, e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, primaryRouteConfigName)},
})
secondaryManagementServer.Update(ctx, e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, secondaryRouteConfigName)},
})
// Configure full resources on tertiary management server.
updateOpts := e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, tertiaryRouteConfigName)},
Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(tertiaryRouteConfigName, serviceName, tertiaryClusterName)},
Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(tertiaryClusterName, tertiaryEndpointsName, e2e.SecurityLevelNone)},
Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(tertiaryEndpointsName, "localhost", []uint32{backend3Port})},
}
tertiaryManagementServer.Update(ctx, updateOpts)
// Create bootstrap configuration for all three management servers.
bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[
{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
},
{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
},
{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}
]`, primaryManagementServer.Address, secondaryManagementServer.Address, tertiaryManagementServer.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
})
if err != nil {
t.Fatalf("Failed to create bootstrap file: %v", err)
}
// Create an xDS client with the above bootstrap configuration.
config, err := bootstrap.NewConfigFromContents(bootstrapContents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %v", err)
}
pool := xdsclient.NewPool(config)
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
// Get the xDS resolver to use the above xDS client.
resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error))
resolver, err := resolverBuilder(pool)
if err != nil {
t.Fatalf("Failed to create xDS resolver for testing: %v", err)
}
// Start a gRPC client that uses the above xDS resolver.
cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
if err != nil {
t.Fatalf("Failed to create gRPC client: %v", err)
}
defer cc.Close()
cc.Connect()
client := testgrpc.NewTestServiceClient(cc)
// Verify that connection attempts were made to primaryWrappedLis and
// secondaryWrappedLis, before using tertiaryWrappedLis to make
// successful RPCs to backend3.
if _, err := primaryWrappedLis.NewConnCh.Receive(ctx); err != nil {
t.Fatalf("Timeout when waiting for connection to primary: %v", err)
}
// Stop primary, client should connect to secondary.
primaryLis.Stop()
if _, err := secondaryWrappedLis.NewConnCh.Receive(ctx); err != nil {
t.Fatalf("Timeout when waiting for connection to secondary after primary stopped: %v", err)
}
// Stop secondary, client should connect to tertiary.
secondaryLis.Stop()
tertiaryConn, err := tertiaryWrappedLis.NewConnCh.Receive(ctx)
if err != nil {
t.Fatalf("Timeout when waiting for connection to tertiary after secondary stopped: %v", err)
}
// Tertiary has all resources, RPCs should succeed to backend3.
if err := waitForRPCsToReachBackend(ctx, client, backend3.Address); err != nil {
t.Fatal(err)
}
// Secondary1 becomes available, RPCs go to backend2.
secondaryLis.Restart()
updateOpts = e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, secondaryRouteConfigName)},
Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(secondaryRouteConfigName, serviceName, secondaryClusterName)},
Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(secondaryClusterName, secondaryEndpointsName, e2e.SecurityLevelNone)},
Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(secondaryEndpointsName, "localhost", []uint32{backend2Port})},
}
secondaryManagementServer.Update(ctx, updateOpts)
secondaryConn, err := secondaryWrappedLis.NewConnCh.Receive(ctx)
if err != nil {
t.Fatalf("Timeout when waiting for new connection to secondary: %v", err)
}
if _, err := tertiaryConn.(*testutils.ConnWrapper).CloseCh.Receive(ctx); err != nil {
t.Fatalf("Timeout when waiting for connection to the tertiary to be closed after promotion to secondary: %v", err)
}
if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil {
t.Fatal(err)
}
// Primary becomes available, RPCs go to backend1.
primaryLis.Restart()
updateOpts = e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, primaryRouteConfigName)},
Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(primaryRouteConfigName, serviceName, primaryClusterName)},
Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(primaryClusterName, primaryEndpointsName, e2e.SecurityLevelNone)},
Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(primaryEndpointsName, "localhost", []uint32{backend1Port})},
}
primaryManagementServer.Update(ctx, updateOpts)
if _, err := primaryWrappedLis.NewConnCh.Receive(ctx); err != nil {
t.Fatalf("Timeout when waiting for new connection to primary: %v", err)
}
if _, err := secondaryConn.(*testutils.ConnWrapper).CloseCh.Receive(ctx); err != nil {
t.Fatalf("Timeout when waiting for connection to the secondary to be closed after promotion to primary: %v", err)
}
if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
t.Fatal(err)
}
}

View File

@@ -0,0 +1,324 @@
/*
*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsclient_test
import (
"context"
"encoding/json"
"fmt"
"testing"
"github.com/google/uuid"
"google.golang.org/grpc/internal/testutils/xds/e2e"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/clients"
"google.golang.org/grpc/internal/xds/xdsclient"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
"google.golang.org/grpc/resolver"
v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
)
const testNonDefaultAuthority = "non-default-authority"
// setupForFederationWatchersTest spins up two management servers, one for the
// default (empty) authority and another for a non-default authority.
//
// Returns the management server associated with the non-default authority, the
// nodeID to use, and the xDS client.
func setupForFederationWatchersTest(t *testing.T) (*e2e.ManagementServer, string, xdsclient.XDSClient) {
// Start a management server as the default authority.
serverDefaultAuthority := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
// Start another management server as the other authority.
serverNonDefaultAuthority := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
nodeID := uuid.New().String()
bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, serverDefaultAuthority.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
Authorities: map[string]json.RawMessage{
testNonDefaultAuthority: []byte(fmt.Sprintf(`{
"xds_servers": [{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]}`, serverNonDefaultAuthority.Address)),
},
})
if err != nil {
t.Fatalf("Failed to create bootstrap configuration: %v", err)
}
// Create an xDS client with the above bootstrap contents.
config, err := bootstrap.NewConfigFromContents(bootstrapContents)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
}
pool := xdsclient.NewPool(config)
client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: t.Name(),
})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
t.Cleanup(close)
return serverNonDefaultAuthority, nodeID, client
}
// TestFederation_ListenerResourceContextParamOrder covers the case of watching
// a Listener resource with the new style resource name and context parameters.
// The test registers watches for two resources which differ only in the order
// of context parameters in their URI. The server is configured to respond with
// a single resource with canonicalized context parameters. The test verifies
// that both watchers are notified.
func (s) TestFederation_ListenerResourceContextParamOrder(t *testing.T) {
serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t)
var (
// Two resource names only differ in context parameter order.
resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/xdsclient-test-lds-resource?a=1&b=2", testNonDefaultAuthority)
resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/xdsclient-test-lds-resource?b=2&a=1", testNonDefaultAuthority)
)
// Register two watches for listener resources with the same query string,
// but context parameters in different order.
lw1 := newListenerWatcher()
ldsCancel1 := xdsresource.WatchListener(client, resourceName1, lw1)
defer ldsCancel1()
lw2 := newListenerWatcher()
ldsCancel2 := xdsresource.WatchListener(client, resourceName2, lw2)
defer ldsCancel2()
// Configure the management server for the non-default authority to return a
// single listener resource, corresponding to the watches registered above.
resources := e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(resourceName1, "rds-resource")},
SkipValidation: true,
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
wantUpdate := listenerUpdateErrTuple{
update: &xdsresource.ListenerUpdate{
APIListener: &xdsresource.HTTPConnectionManagerConfig{
RouteConfigName: "rds-resource",
HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}},
},
},
}
// Verify the contents of the received update.
if err := verifyListenerUpdate(ctx, lw1.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
}
// TestFederation_RouteConfigResourceContextParamOrder covers the case of
// watching a RouteConfiguration resource with the new style resource name and
// context parameters. The test registers watches for two resources which
// differ only in the order of context parameters in their URI. The server is
// configured to respond with a single resource with canonicalized context
// parameters. The test verifies that both watchers are notified.
func (s) TestFederation_RouteConfigResourceContextParamOrder(t *testing.T) {
serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t)
var (
// Two resource names only differ in context parameter order.
resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource?a=1&b=2", testNonDefaultAuthority)
resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource?b=2&a=1", testNonDefaultAuthority)
)
// Register two watches for route configuration resources with the same
// query string, but context parameters in different order.
rw1 := newRouteConfigWatcher()
rdsCancel1 := xdsresource.WatchRouteConfig(client, resourceName1, rw1)
defer rdsCancel1()
rw2 := newRouteConfigWatcher()
rdsCancel2 := xdsresource.WatchRouteConfig(client, resourceName2, rw2)
defer rdsCancel2()
// Configure the management server for the non-default authority to return a
// single route config resource, corresponding to the watches registered.
resources := e2e.UpdateOptions{
NodeID: nodeID,
Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(resourceName1, "listener-resource", "cluster-resource")},
SkipValidation: true,
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
wantUpdate := routeConfigUpdateErrTuple{
update: xdsresource.RouteConfigUpdate{
VirtualHosts: []*xdsresource.VirtualHost{
{
Domains: []string{"listener-resource"},
Routes: []*xdsresource.Route{
{
Prefix: newStringP("/"),
ActionType: xdsresource.RouteActionRoute,
WeightedClusters: []xdsresource.WeightedCluster{{Name: "cluster-resource", Weight: 100}},
},
},
},
},
},
}
// Verify the contents of the received update.
if err := verifyRouteConfigUpdate(ctx, rw1.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
if err := verifyRouteConfigUpdate(ctx, rw2.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
}
// TestFederation_ClusterResourceContextParamOrder covers the case of watching a
// Cluster resource with the new style resource name and context parameters.
// The test registers watches for two resources which differ only in the order
// of context parameters in their URI. The server is configured to respond with
// a single resource with canonicalized context parameters. The test verifies
// that both watchers are notified.
func (s) TestFederation_ClusterResourceContextParamOrder(t *testing.T) {
serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t)
var (
// Two resource names only differ in context parameter order.
resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource?a=1&b=2", testNonDefaultAuthority)
resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource?b=2&a=1", testNonDefaultAuthority)
)
// Register two watches for cluster resources with the same query string,
// but context parameters in different order.
cw1 := newClusterWatcher()
cdsCancel1 := xdsresource.WatchCluster(client, resourceName1, cw1)
defer cdsCancel1()
cw2 := newClusterWatcher()
cdsCancel2 := xdsresource.WatchCluster(client, resourceName2, cw2)
defer cdsCancel2()
// Configure the management server for the non-default authority to return a
// single cluster resource, corresponding to the watches registered.
resources := e2e.UpdateOptions{
NodeID: nodeID,
Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(resourceName1, "eds-service-name", e2e.SecurityLevelNone)},
SkipValidation: true,
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
wantUpdate := clusterUpdateErrTuple{
update: xdsresource.ClusterUpdate{
ClusterName: "xdstp://non-default-authority/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource?a=1&b=2",
EDSServiceName: "eds-service-name",
},
}
// Verify the contents of the received update.
if err := verifyClusterUpdate(ctx, cw1.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
if err := verifyClusterUpdate(ctx, cw2.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
}
// TestFederation_EndpointsResourceContextParamOrder covers the case of watching
// an Endpoints resource with the new style resource name and context parameters.
// The test registers watches for two resources which differ only in the order
// of context parameters in their URI. The server is configured to respond with
// a single resource with canonicalized context parameters. The test verifies
// that both watchers are notified.
func (s) TestFederation_EndpointsResourceContextParamOrder(t *testing.T) {
serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t)
var (
// Two resource names only differ in context parameter order.
resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource?a=1&b=2", testNonDefaultAuthority)
resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource?b=2&a=1", testNonDefaultAuthority)
)
// Register two watches for endpoint resources with the same query string,
// but context parameters in different order.
ew1 := newEndpointsWatcher()
edsCancel1 := xdsresource.WatchEndpoints(client, resourceName1, ew1)
defer edsCancel1()
ew2 := newEndpointsWatcher()
edsCancel2 := xdsresource.WatchEndpoints(client, resourceName2, ew2)
defer edsCancel2()
// Configure the management server for the non-default authority to return a
// single endpoints resource, corresponding to the watches registered.
resources := e2e.UpdateOptions{
NodeID: nodeID,
Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(resourceName1, "localhost", []uint32{666})},
SkipValidation: true,
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
wantUpdate := endpointsUpdateErrTuple{
update: xdsresource.EndpointsUpdate{
Localities: []xdsresource.Locality{
{
Endpoints: []xdsresource.Endpoint{{
ResolverEndpoint: resolver.Endpoint{
Addresses: []resolver.Address{{Addr: "localhost:666"}},
},
Weight: 1,
}},
Weight: 1,
ID: clients.Locality{
Region: "region-1",
Zone: "zone-1",
SubZone: "subzone-1",
},
},
},
},
}
// Verify the contents of the received update.
if err := verifyEndpointsUpdate(ctx, ew1.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
if err := verifyEndpointsUpdate(ctx, ew2.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
}
func newStringP(s string) *string {
return &s
}

View File

@@ -0,0 +1,72 @@
/*
*
* Copyright 2024 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient_test
import (
"fmt"
"strings"
"testing"
"time"
"google.golang.org/grpc/internal/grpctest"
)
type s struct {
grpctest.Tester
}
func Test(t *testing.T) {
grpctest.RunSubTests(t, s{})
}
const (
defaultTestWatchExpiryTimeout = 500 * time.Millisecond
defaultTestTimeout = 10 * time.Second
defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen.
ldsName = "xdsclient-test-lds-resource"
rdsName = "xdsclient-test-rds-resource"
cdsName = "xdsclient-test-cds-resource"
edsName = "xdsclient-test-eds-resource"
ldsNameNewStyle = "xdstp:///envoy.config.listener.v3.Listener/xdsclient-test-lds-resource"
rdsNameNewStyle = "xdstp:///envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource"
cdsNameNewStyle = "xdstp:///envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource"
edsNameNewStyle = "xdstp:///envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource"
)
func makeAuthorityName(name string) string {
segs := strings.Split(name, "/")
return strings.Join(segs, "")
}
func makeNewStyleLDSName(authority string) string {
return fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/xdsclient-test-lds-resource", authority)
}
func makeNewStyleRDSName(authority string) string {
return fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource", authority)
}
func makeNewStyleCDSName(authority string) string {
return fmt.Sprintf("xdstp://%s/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource", authority)
}
func makeNewStyleEDSName(authority string) string {
return fmt.Sprintf("xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource", authority)
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,566 @@
/*
*
* Copyright 2024 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient_test
import (
"context"
"encoding/json"
"fmt"
"net"
"sync"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/google/uuid"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/internal"
"google.golang.org/grpc/internal/stubserver"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/testutils/xds/e2e"
"google.golang.org/grpc/internal/testutils/xds/fakeserver"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/clients"
"google.golang.org/grpc/internal/xds/xdsclient"
"google.golang.org/grpc/resolver"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/testing/protocmp"
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3"
testgrpc "google.golang.org/grpc/interop/grpc_testing"
testpb "google.golang.org/grpc/interop/grpc_testing"
"google.golang.org/protobuf/types/known/durationpb"
)
const (
testKey1 = "test-key1"
testKey2 = "test-key2"
)
var (
testLocality1 = clients.Locality{Region: "test-region1"}
testLocality2 = clients.Locality{Region: "test-region2"}
toleranceCmpOpt = cmpopts.EquateApprox(0, 1e-5)
ignoreOrderCmpOpt = protocmp.FilterField(&v3endpointpb.ClusterStats{}, "upstream_locality_stats",
cmpopts.SortSlices(func(a, b protocmp.Message) bool {
return a.String() < b.String()
}),
)
)
type wrappedListener struct {
net.Listener
newConnChan *testutils.Channel // Connection attempts are pushed here.
}
func (wl *wrappedListener) Accept() (net.Conn, error) {
c, err := wl.Listener.Accept()
if err != nil {
return nil, err
}
wl.newConnChan.Send(struct{}{})
return c, err
}
// Tests a load reporting scenario where the xDS client is reporting loads to
// multiple servers. Verifies the following:
// - calling the load reporting API with different server configuration
// results in connections being created to those corresponding servers
// - the same load.Store is not returned when the load reporting API called
// with different server configurations
// - canceling the load reporting from the client results in the LRS stream
// being canceled on the server
func (s) TestReportLoad_ConnectionCreation(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
// Create two management servers that also serve LRS.
l, err := testutils.LocalTCPListener()
if err != nil {
t.Fatalf("Failed to create a local TCP listener: %v", err)
}
newConnChan1 := testutils.NewChannel()
lis1 := &wrappedListener{
Listener: l,
newConnChan: newConnChan1,
}
mgmtServer1 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
Listener: lis1,
SupportLoadReportingService: true,
})
l, err = testutils.LocalTCPListener()
if err != nil {
t.Fatalf("Failed to create a local TCP listener: %v", err)
}
newConnChan2 := testutils.NewChannel()
lis2 := &wrappedListener{
Listener: l,
newConnChan: newConnChan2,
}
mgmtServer2 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
Listener: lis2,
SupportLoadReportingService: true,
})
// Create an xDS client with a bootstrap configuration that contains both of
// the above two servers. The authority name is immaterial here since load
// reporting is per-server and not per-authority.
nodeID := uuid.New().String()
bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, mgmtServer1.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
Authorities: map[string]json.RawMessage{
"test-authority": []byte(fmt.Sprintf(`{
"xds_servers": [{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]}`, mgmtServer2.Address)),
},
})
if err != nil {
t.Fatalf("Failed to create bootstrap configuration: %v", err)
}
client := createXDSClient(t, bc)
serverCfg1, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: mgmtServer1.Address})
if err != nil {
t.Fatalf("Failed to create server config for testing: %v", err)
}
// Call the load reporting API to report load to the first management
// server, and ensure that a connection to the server is created.
store1, lrsCancel1 := client.ReportLoad(serverCfg1)
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
defer lrsCancel1(sCtx)
if _, err := newConnChan1.Receive(ctx); err != nil {
t.Fatal("Timeout when waiting for a connection to the first management server, after starting load reporting")
}
if _, err := mgmtServer1.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil {
t.Fatal("Timeout when waiting for LRS stream to be created")
}
serverCfg2, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: mgmtServer2.Address})
if err != nil {
t.Fatalf("Failed to create server config for testing: %v", err)
}
// Call the load reporting API to report load to the second management
// server, and ensure that a connection to the server is created.
store2, lrsCancel2 := client.ReportLoad(serverCfg2)
sCtx2, sCancel2 := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel2()
defer lrsCancel2(sCtx2)
if _, err := newConnChan2.Receive(ctx); err != nil {
t.Fatal("Timeout when waiting for a connection to the second management server, after starting load reporting")
}
if _, err := mgmtServer2.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil {
t.Fatal("Timeout when waiting for LRS stream to be created")
}
if store1 == store2 {
t.Fatalf("Got same store for different servers, want different")
}
// Push some loads on the received store.
store2.ReporterForCluster("cluster", "eds").CallDropped("test")
// Ensure the initial load reporting request is received at the server.
lrsServer := mgmtServer2.LRSServer
req, err := lrsServer.LRSRequestChan.Receive(ctx)
if err != nil {
t.Fatalf("Timeout when waiting for initial LRS request: %v", err)
}
gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest)
nodeProto := &v3corepb.Node{
Id: nodeID,
UserAgentName: "gRPC Go",
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"},
}
wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto}
if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" {
t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff)
}
// Send a response from the server with a small deadline.
lrsServer.LRSResponseChan <- &fakeserver.Response{
Resp: &v3lrspb.LoadStatsResponse{
SendAllClusters: true,
LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms
},
}
// Ensure that loads are seen on the server.
req, err = lrsServer.LRSRequestChan.Receive(ctx)
if err != nil {
t.Fatalf("Timeout when waiting for LRS request with loads: %v", err)
}
gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
if l := len(gotLoad); l != 1 {
t.Fatalf("Received load for %d clusters, want 1", l)
}
// This field is set by the client to indicate the actual time elapsed since
// the last report was sent. We cannot deterministically compare this, and
// we cannot use the cmpopts.IgnoreFields() option on proto structs, since
// we already use the protocmp.Transform() which marshals the struct into
// another message. Hence setting this field to nil is the best option here.
gotLoad[0].LoadReportInterval = nil
wantLoad := &v3endpointpb.ClusterStats{
ClusterName: "cluster",
ClusterServiceName: "eds",
TotalDroppedRequests: 1,
DroppedRequests: []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
}
if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform(), toleranceCmpOpt, ignoreOrderCmpOpt); diff != "" {
t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
}
// Cancel this load reporting stream, server should see error canceled.
sCtx2, sCancel2 = context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel2()
lrsCancel2(sCtx2)
// Server should receive a stream canceled error. There may be additional
// load reports from the client in the channel.
for {
if ctx.Err() != nil {
t.Fatal("Timeout when waiting for the LRS stream to be canceled on the server")
}
u, err := lrsServer.LRSRequestChan.Receive(ctx)
if err != nil {
continue
}
// Ignore load reports sent before the stream was cancelled.
if u.(*fakeserver.Request).Err == nil {
continue
}
if status.Code(u.(*fakeserver.Request).Err) != codes.Canceled {
t.Fatalf("Unexpected LRS request: %v, want error canceled", u)
}
break
}
}
// Tests a load reporting scenario where the load reporting API is called
// multiple times for the same server. The test verifies the following:
// - calling the load reporting API the second time for the same server
// configuration does not create a new LRS stream
// - the LRS stream is closed *only* after all the API calls invoke their
// cancel functions
// - creating new streams after the previous one was closed works
func (s) TestReportLoad_StreamCreation(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
// Create a management server that serves LRS.
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
// Create an xDS client with bootstrap pointing to the above server.
nodeID := uuid.New().String()
bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
client := createXDSClient(t, bc)
// Call the load reporting API, and ensure that an LRS stream is created.
serverConfig, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: mgmtServer.Address})
if err != nil {
t.Fatalf("Failed to create server config for testing: %v", err)
}
store1, cancel1 := client.ReportLoad(serverConfig)
lrsServer := mgmtServer.LRSServer
if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil {
t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err)
}
// Push some loads on the received store.
store1.ReporterForCluster("cluster1", "eds1").CallDropped("test")
store1.ReporterForCluster("cluster1", "eds1").CallStarted(testLocality1)
store1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, testKey1, 3.14)
store1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, testKey1, 2.718)
store1.ReporterForCluster("cluster1", "eds1").CallFinished(testLocality1, nil)
store1.ReporterForCluster("cluster1", "eds1").CallStarted(testLocality2)
store1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality2, testKey2, 1.618)
store1.ReporterForCluster("cluster1", "eds1").CallFinished(testLocality2, nil)
// Ensure the initial load reporting request is received at the server.
req, err := lrsServer.LRSRequestChan.Receive(ctx)
if err != nil {
t.Fatalf("Timeout when waiting for initial LRS request: %v", err)
}
gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest)
nodeProto := &v3corepb.Node{
Id: nodeID,
UserAgentName: "gRPC Go",
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"},
}
wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto}
if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" {
t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff)
}
// Send a response from the server with a small deadline.
lrsServer.LRSResponseChan <- &fakeserver.Response{
Resp: &v3lrspb.LoadStatsResponse{
SendAllClusters: true,
LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms
},
}
// Ensure that loads are seen on the server.
req, err = lrsServer.LRSRequestChan.Receive(ctx)
if err != nil {
t.Fatal("Timeout when waiting for LRS request with loads")
}
gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
if l := len(gotLoad); l != 1 {
t.Fatalf("Received load for %d clusters, want 1", l)
}
// This field is set by the client to indicate the actual time elapsed since
// the last report was sent. We cannot deterministically compare this, and
// we cannot use the cmpopts.IgnoreFields() option on proto structs, since
// we already use the protocmp.Transform() which marshals the struct into
// another message. Hence setting this field to nil is the best option here.
gotLoad[0].LoadReportInterval = nil
wantLoad := &v3endpointpb.ClusterStats{
ClusterName: "cluster1",
ClusterServiceName: "eds1",
TotalDroppedRequests: 1,
DroppedRequests: []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
UpstreamLocalityStats: []*v3endpointpb.UpstreamLocalityStats{
{
Locality: &v3corepb.Locality{Region: "test-region1"},
LoadMetricStats: []*v3endpointpb.EndpointLoadMetricStats{
// TotalMetricValue is the aggregation of 3.14 + 2.718 = 5.858
{MetricName: testKey1, NumRequestsFinishedWithMetric: 2, TotalMetricValue: 5.858}},
TotalSuccessfulRequests: 1,
TotalIssuedRequests: 1,
},
{
Locality: &v3corepb.Locality{Region: "test-region2"},
LoadMetricStats: []*v3endpointpb.EndpointLoadMetricStats{
{MetricName: testKey2, NumRequestsFinishedWithMetric: 1, TotalMetricValue: 1.618}},
TotalSuccessfulRequests: 1,
TotalIssuedRequests: 1,
},
},
}
if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform(), toleranceCmpOpt, ignoreOrderCmpOpt); diff != "" {
t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
}
// Make another call to the load reporting API, and ensure that a new LRS
// stream is not created.
store2, cancel2 := client.ReportLoad(serverConfig)
sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
defer sCancel()
if _, err := lrsServer.LRSStreamOpenChan.Receive(sCtx); err != context.DeadlineExceeded {
t.Fatal("New LRS stream created when expected to use an existing one")
}
// Push more loads.
store2.ReporterForCluster("cluster2", "eds2").CallDropped("test")
// Ensure that loads are seen on the server. We need a loop here because
// there could have been some requests from the client in the time between
// us reading the first request and now. Those would have been queued in the
// request channel that we read out of.
for {
if ctx.Err() != nil {
t.Fatalf("Timeout when waiting for new loads to be seen on the server")
}
req, err = lrsServer.LRSRequestChan.Receive(ctx)
if err != nil {
continue
}
gotLoad = req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
if l := len(gotLoad); l != 1 {
continue
}
gotLoad[0].LoadReportInterval = nil
wantLoad := &v3endpointpb.ClusterStats{
ClusterName: "cluster2",
ClusterServiceName: "eds2",
TotalDroppedRequests: 1,
DroppedRequests: []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
}
if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform()); diff != "" {
t.Logf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
continue
}
break
}
// Cancel the first load reporting call, and ensure that the stream does not
// close (because we have another call open).
sCtx1, sCancel1 := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel1()
cancel1(sCtx1)
sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
defer sCancel()
if _, err := lrsServer.LRSStreamCloseChan.Receive(sCtx); err != context.DeadlineExceeded {
t.Fatal("LRS stream closed when expected to stay open")
}
// Cancel the second load reporting call, and ensure the stream is closed.
sCtx2, sCancel2 := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel2()
cancel2(sCtx2)
if _, err := lrsServer.LRSStreamCloseChan.Receive(ctx); err != nil {
t.Fatal("Timeout waiting for LRS stream to close")
}
// Calling the load reporting API again should result in the creation of a
// new LRS stream. This ensures that creating and closing multiple streams
// works smoothly.
_, cancel3 := client.ReportLoad(serverConfig)
if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil {
t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err)
}
sCtx3, sCancel3 := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel3()
cancel3(sCtx3)
}
// TestConcurrentReportLoad verifies that the client can safely handle concurrent
// requests to initiate load reporting streams. It launches multiple goroutines
// that all call client.ReportLoad simultaneously. We don't verify anything on
// the server here, since that is done in other tests. This is just to ensure
// that concurrent ReportLoad() calls work.
func (s) TestConcurrentReportLoad(t *testing.T) {
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
nodeID := uuid.New().String()
bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
client := createXDSClient(t, bc)
serverConfig, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: mgmtServer.Address})
if err != nil {
t.Fatalf("Failed to create server config for testing: %v", err)
}
// Call ReportLoad() concurrently from multiple go routines.
const numGoroutines = 10
cancelStores := make([]func(context.Context), numGoroutines)
var wg sync.WaitGroup
wg.Add(numGoroutines)
for i := range numGoroutines {
go func(idx int) {
defer wg.Done()
_, cancelStores[idx] = client.ReportLoad(serverConfig)
}(i)
}
wg.Wait()
// Cancel all the load reporting streams. The last call to cancel is
// expected to block until a final load report with pending loads is sent.
// But the stream is currently blocked on a recv() call waiting for the LRS
// server to send the initial laod reporting response, which it never does.
// Hence we use a context with short timeout here.
for i, cancel := range cancelStores {
sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
defer sCancel()
cancel(sCtx)
t.Logf("Cancelled LRS stream %d", i)
}
}
// TestConcurrentChannels verifies that we can create multiple gRPC channels
// concurrently with a shared XDSClient, each of which will create a new LRS
// stream without any race.
func (s) TestConcurrentChannels(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true, SupportLoadReportingService: true})
nodeID := uuid.New().String()
bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
if internal.NewXDSResolverWithPoolForTesting == nil {
t.Fatalf("internal.NewXDSResolverWithConfigForTesting is nil")
}
config, err := bootstrap.NewConfigFromContents(bc)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
}
pool := xdsclient.NewPool(config)
resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error))
xdsResolver, err := resolverBuilder(pool)
if err != nil {
t.Fatalf("Failed to create xDS resolver for testing: %v", err)
}
server := stubserver.StartTestService(t, nil)
defer server.Stop()
// Configure the management server with resources that enable LRS.
const serviceName = "my-service-e2e-lrs-test"
resources := e2e.DefaultClientResources(e2e.ResourceParams{
DialTarget: serviceName,
NodeID: nodeID,
Host: "localhost",
Port: testutils.ParsePort(t, server.Address),
SecLevel: e2e.SecurityLevelNone,
})
resources.Clusters[0].LrsServer = &v3corepb.ConfigSource{
ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{
Self: &v3corepb.SelfConfigSource{},
},
}
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatal(err)
}
var wg sync.WaitGroup
const (
numGoroutines = 10
numRPCs = 10
)
for range numGoroutines {
wg.Add(1)
go func() {
defer wg.Done()
for range numRPCs {
cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
if err != nil {
t.Errorf("grpc.NewClient() failed: %v", err)
return
}
defer cc.Close()
testClient := testgrpc.NewTestServiceClient(cc)
if _, err := testClient.EmptyCall(ctx, &testpb.Empty{}); err != nil {
t.Errorf("EmptyCall() failed: %v", err)
return
}
}
}()
}
wg.Wait()
}

View File

@@ -0,0 +1,996 @@
/*
*
* Copyright 2022 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient_test
import (
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/google/uuid"
"google.golang.org/grpc/internal/grpcsync"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/testutils/xds/e2e"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/xdsclient"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
"google.golang.org/protobuf/types/known/wrapperspb"
v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
)
type noopRouteConfigWatcher struct{}
func (noopRouteConfigWatcher) ResourceChanged(_ *xdsresource.RouteConfigUpdate, onDone func()) {
onDone()
}
func (noopRouteConfigWatcher) ResourceError(_ error, onDone func()) {
onDone()
}
func (noopRouteConfigWatcher) AmbientError(_ error, onDone func()) {
onDone()
}
type routeConfigUpdateErrTuple struct {
update xdsresource.RouteConfigUpdate
err error
}
type routeConfigWatcher struct {
updateCh *testutils.Channel
}
func newRouteConfigWatcher() *routeConfigWatcher {
return &routeConfigWatcher{updateCh: testutils.NewChannel()}
}
func (rw *routeConfigWatcher) ResourceChanged(update *xdsresource.RouteConfigUpdate, onDone func()) {
rw.updateCh.Send(routeConfigUpdateErrTuple{update: *update})
onDone()
}
func (rw *routeConfigWatcher) ResourceError(err error, onDone func()) {
// When used with a go-control-plane management server that continuously
// resends resources which are NACKed by the xDS client, using a `Replace()`
// here and in AmbientError() simplifies tests which will have
// access to the most recently received error.
rw.updateCh.Replace(routeConfigUpdateErrTuple{err: err})
onDone()
}
func (rw *routeConfigWatcher) AmbientError(err error, onDone func()) {
rw.updateCh.Replace(routeConfigUpdateErrTuple{err: err})
onDone()
}
// badRouteConfigResource returns a RouteConfiguration resource for the given
// routeName which contains a retry config with num_retries set to `0`. This is
// expected to be NACK'ed by the xDS client.
func badRouteConfigResource(routeName, ldsTarget, clusterName string) *v3routepb.RouteConfiguration {
return &v3routepb.RouteConfiguration{
Name: routeName,
VirtualHosts: []*v3routepb.VirtualHost{{
Domains: []string{ldsTarget},
Routes: []*v3routepb.Route{{
Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}},
Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName},
}}}},
RetryPolicy: &v3routepb.RetryPolicy{
NumRetries: &wrapperspb.UInt32Value{Value: 0},
},
}},
}
}
// xdsClient is expected to produce an error containing this string when an
// update is received containing a route configuration resource created using
// `badRouteConfigResource`.
const wantRouteConfigNACKErr = "received route is invalid: retry_policy.num_retries = 0; must be >= 1"
// verifyRouteConfigUpdate waits for an update to be received on the provided
// update channel and verifies that it matches the expected update.
//
// Returns an error if no update is received before the context deadline expires
// or the received update does not match the expected one.
func verifyRouteConfigUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate routeConfigUpdateErrTuple) error {
u, err := updateCh.Receive(ctx)
if err != nil {
return fmt.Errorf("timeout when waiting for a route configuration resource from the management server: %v", err)
}
got := u.(routeConfigUpdateErrTuple)
if wantUpdate.err != nil {
if got.err == nil || !strings.Contains(got.err.Error(), wantUpdate.err.Error()) {
return fmt.Errorf("update received with error: %v, want %q", got.err, wantUpdate.err)
}
}
cmpOpts := []cmp.Option{cmpopts.EquateEmpty(), cmpopts.IgnoreFields(xdsresource.RouteConfigUpdate{}, "Raw")}
if diff := cmp.Diff(wantUpdate.update, got.update, cmpOpts...); diff != "" {
return fmt.Errorf("received unexpected diff in the route configuration resource update: (-want, got):\n%s", diff)
}
return nil
}
// verifyNoRouteConfigUpdate verifies that no route configuration update is
// received on the provided update channel, and returns an error if an update is
// received.
//
// A very short deadline is used while waiting for the update, as this function
// is intended to be used when an update is not expected.
func verifyNoRouteConfigUpdate(ctx context.Context, updateCh *testutils.Channel) error {
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
if u, err := updateCh.Receive(sCtx); err != context.DeadlineExceeded {
return fmt.Errorf("unexpected RouteConfigUpdate: %v", u)
}
return nil
}
// TestRDSWatch covers the case where a single watcher exists for a single route
// configuration resource. The test verifies the following scenarios:
// 1. An update from the management server containing the resource being
// watched should result in the invocation of the watch callback.
// 2. An update from the management server containing a resource *not* being
// watched should not result in the invocation of the watch callback.
// 3. After the watch is cancelled, an update from the management server
// containing the resource that was being watched should not result in the
// invocation of the watch callback.
//
// The test is run for old and new style names.
func (s) TestRDSWatch(t *testing.T) {
tests := []struct {
desc string
resourceName string
watchedResource *v3routepb.RouteConfiguration // The resource being watched.
updatedWatchedResource *v3routepb.RouteConfiguration // The watched resource after an update.
notWatchedResource *v3routepb.RouteConfiguration // A resource which is not being watched.
wantUpdate routeConfigUpdateErrTuple
}{
{
desc: "old style resource",
resourceName: rdsName,
watchedResource: e2e.DefaultRouteConfig(rdsName, ldsName, cdsName),
updatedWatchedResource: e2e.DefaultRouteConfig(rdsName, ldsName, "new-cds-resource"),
notWatchedResource: e2e.DefaultRouteConfig("unsubscribed-rds-resource", ldsName, cdsName),
wantUpdate: routeConfigUpdateErrTuple{
update: xdsresource.RouteConfigUpdate{
VirtualHosts: []*xdsresource.VirtualHost{
{
Domains: []string{ldsName},
Routes: []*xdsresource.Route{
{
Prefix: newStringP("/"),
ActionType: xdsresource.RouteActionRoute,
WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}},
},
},
},
},
},
},
},
{
desc: "new style resource",
resourceName: rdsNameNewStyle,
watchedResource: e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, cdsNameNewStyle),
updatedWatchedResource: e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, "new-cds-resource"),
notWatchedResource: e2e.DefaultRouteConfig("unsubscribed-rds-resource", ldsNameNewStyle, cdsNameNewStyle),
wantUpdate: routeConfigUpdateErrTuple{
update: xdsresource.RouteConfigUpdate{
VirtualHosts: []*xdsresource.VirtualHost{
{
Domains: []string{ldsNameNewStyle},
Routes: []*xdsresource.Route{
{
Prefix: newStringP("/"),
ActionType: xdsresource.RouteActionRoute,
WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsNameNewStyle, Weight: 100}},
},
},
},
},
},
},
},
}
for _, test := range tests {
t.Run(test.desc, func(t *testing.T) {
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
nodeID := uuid.New().String()
bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, mgmtServer.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
Authorities: map[string]json.RawMessage{
// Xdstp resource names used in this test do not specify an
// authority. These will end up looking up an entry with the
// empty key in the authorities map. Having an entry with an
// empty key and empty configuration, results in these
// resources also using the top-level configuration.
"": []byte(`{}`),
},
})
if err != nil {
t.Fatalf("Failed to create bootstrap configuration: %v", err)
}
// Create an xDS client with the above bootstrap contents.
config, err := bootstrap.NewConfigFromContents(bc)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
}
pool := xdsclient.NewPool(config)
client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: t.Name(),
})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
defer close()
// Register a watch for a route configuration resource and have the
// watch callback push the received update on to a channel.
rw := newRouteConfigWatcher()
rdsCancel := xdsresource.WatchRouteConfig(client, test.resourceName, rw)
// Configure the management server to return a single route
// configuration resource, corresponding to the one being watched.
resources := e2e.UpdateOptions{
NodeID: nodeID,
Routes: []*v3routepb.RouteConfiguration{test.watchedResource},
SkipValidation: true,
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
// Verify the contents of the received update.
if err := verifyRouteConfigUpdate(ctx, rw.updateCh, test.wantUpdate); err != nil {
t.Fatal(err)
}
// Configure the management server to return an additional route
// configuration resource, one that we are not interested in.
resources = e2e.UpdateOptions{
NodeID: nodeID,
Routes: []*v3routepb.RouteConfiguration{test.watchedResource, test.notWatchedResource},
SkipValidation: true,
}
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
if err := verifyNoRouteConfigUpdate(ctx, rw.updateCh); err != nil {
t.Fatal(err)
}
// Cancel the watch and update the resource corresponding to the original
// watch. Ensure that the cancelled watch callback is not invoked.
rdsCancel()
resources = e2e.UpdateOptions{
NodeID: nodeID,
Routes: []*v3routepb.RouteConfiguration{test.updatedWatchedResource, test.notWatchedResource},
SkipValidation: true,
}
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
if err := verifyNoRouteConfigUpdate(ctx, rw.updateCh); err != nil {
t.Fatal(err)
}
})
}
}
// TestRDSWatch_TwoWatchesForSameResourceName covers the case where two watchers
// exist for a single route configuration resource. The test verifies the
// following scenarios:
// 1. An update from the management server containing the resource being
// watched should result in the invocation of both watch callbacks.
// 2. After one of the watches is cancelled, a redundant update from the
// management server should not result in the invocation of either of the
// watch callbacks.
// 3. An update from the management server containing the resource being
// watched should result in the invocation of the un-cancelled watch
// callback.
//
// The test is run for old and new style names.
func (s) TestRDSWatch_TwoWatchesForSameResourceName(t *testing.T) {
tests := []struct {
desc string
resourceName string
watchedResource *v3routepb.RouteConfiguration // The resource being watched.
updatedWatchedResource *v3routepb.RouteConfiguration // The watched resource after an update.
wantUpdateV1 routeConfigUpdateErrTuple
wantUpdateV2 routeConfigUpdateErrTuple
}{
{
desc: "old style resource",
resourceName: rdsName,
watchedResource: e2e.DefaultRouteConfig(rdsName, ldsName, cdsName),
updatedWatchedResource: e2e.DefaultRouteConfig(rdsName, ldsName, "new-cds-resource"),
wantUpdateV1: routeConfigUpdateErrTuple{
update: xdsresource.RouteConfigUpdate{
VirtualHosts: []*xdsresource.VirtualHost{
{
Domains: []string{ldsName},
Routes: []*xdsresource.Route{
{
Prefix: newStringP("/"),
ActionType: xdsresource.RouteActionRoute,
WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}},
},
},
},
},
},
},
wantUpdateV2: routeConfigUpdateErrTuple{
update: xdsresource.RouteConfigUpdate{
VirtualHosts: []*xdsresource.VirtualHost{
{
Domains: []string{ldsName},
Routes: []*xdsresource.Route{
{
Prefix: newStringP("/"),
ActionType: xdsresource.RouteActionRoute,
WeightedClusters: []xdsresource.WeightedCluster{{Name: "new-cds-resource", Weight: 100}},
},
},
},
},
},
},
},
{
desc: "new style resource",
resourceName: rdsNameNewStyle,
watchedResource: e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, cdsNameNewStyle),
updatedWatchedResource: e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, "new-cds-resource"),
wantUpdateV1: routeConfigUpdateErrTuple{
update: xdsresource.RouteConfigUpdate{
VirtualHosts: []*xdsresource.VirtualHost{
{
Domains: []string{ldsNameNewStyle},
Routes: []*xdsresource.Route{
{
Prefix: newStringP("/"),
ActionType: xdsresource.RouteActionRoute,
WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsNameNewStyle, Weight: 100}},
},
},
},
},
},
},
wantUpdateV2: routeConfigUpdateErrTuple{
update: xdsresource.RouteConfigUpdate{
VirtualHosts: []*xdsresource.VirtualHost{
{
Domains: []string{ldsNameNewStyle},
Routes: []*xdsresource.Route{
{
Prefix: newStringP("/"),
ActionType: xdsresource.RouteActionRoute,
WeightedClusters: []xdsresource.WeightedCluster{{Name: "new-cds-resource", Weight: 100}},
},
},
},
},
},
},
},
}
for _, test := range tests {
t.Run(test.desc, func(t *testing.T) {
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
nodeID := uuid.New().String()
bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, mgmtServer.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
Authorities: map[string]json.RawMessage{
// Xdstp resource names used in this test do not specify an
// authority. These will end up looking up an entry with the
// empty key in the authorities map. Having an entry with an
// empty key and empty configuration, results in these
// resources also using the top-level configuration.
"": []byte(`{}`),
},
})
if err != nil {
t.Fatalf("Failed to create bootstrap configuration: %v", err)
}
// Create an xDS client with the above bootstrap contents.
config, err := bootstrap.NewConfigFromContents(bc)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
}
pool := xdsclient.NewPool(config)
client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: t.Name(),
})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
defer close()
// Register two watches for the same route configuration resource
// and have the callbacks push the received updates on to a channel.
rw1 := newRouteConfigWatcher()
rdsCancel1 := xdsresource.WatchRouteConfig(client, test.resourceName, rw1)
defer rdsCancel1()
rw2 := newRouteConfigWatcher()
rdsCancel2 := xdsresource.WatchRouteConfig(client, test.resourceName, rw2)
// Configure the management server to return a single route
// configuration resource, corresponding to the one being watched.
resources := e2e.UpdateOptions{
NodeID: nodeID,
Routes: []*v3routepb.RouteConfiguration{test.watchedResource},
SkipValidation: true,
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
// Verify the contents of the received update.
if err := verifyRouteConfigUpdate(ctx, rw1.updateCh, test.wantUpdateV1); err != nil {
t.Fatal(err)
}
if err := verifyRouteConfigUpdate(ctx, rw2.updateCh, test.wantUpdateV1); err != nil {
t.Fatal(err)
}
// Cancel the second watch and force the management server to push a
// redundant update for the resource being watched. Neither of the
// two watch callbacks should be invoked.
rdsCancel2()
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
if err := verifyNoRouteConfigUpdate(ctx, rw1.updateCh); err != nil {
t.Fatal(err)
}
if err := verifyNoRouteConfigUpdate(ctx, rw2.updateCh); err != nil {
t.Fatal(err)
}
// Update to the resource being watched. The un-cancelled callback
// should be invoked while the cancelled one should not be.
resources = e2e.UpdateOptions{
NodeID: nodeID,
Routes: []*v3routepb.RouteConfiguration{test.updatedWatchedResource},
SkipValidation: true,
}
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
if err := verifyRouteConfigUpdate(ctx, rw1.updateCh, test.wantUpdateV2); err != nil {
t.Fatal(err)
}
if err := verifyNoRouteConfigUpdate(ctx, rw2.updateCh); err != nil {
t.Fatal(err)
}
})
}
}
// TestRDSWatch_ThreeWatchesForDifferentResourceNames covers the case with three
// watchers (two watchers for one resource, and the third watcher for another
// resource), exist across two route configuration resources. The test verifies
// that an update from the management server containing both resources results
// in the invocation of all watch callbacks.
//
// The test is run with both old and new style names.
func (s) TestRDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) {
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
nodeID := uuid.New().String()
authority := makeAuthorityName(t.Name())
bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, mgmtServer.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
Authorities: map[string]json.RawMessage{
// Xdstp style resource names used in this test use a slash removed
// version of t.Name as their authority, and the empty config
// results in the top-level xds server configuration being used for
// this authority.
authority: []byte(`{}`),
},
})
if err != nil {
t.Fatalf("Failed to create bootstrap configuration: %v", err)
}
// Create an xDS client with the above bootstrap contents.
config, err := bootstrap.NewConfigFromContents(bc)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
}
pool := xdsclient.NewPool(config)
client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: t.Name(),
})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
defer close()
// Register two watches for the same route configuration resource
// and have the callbacks push the received updates on to a channel.
rw1 := newRouteConfigWatcher()
rdsCancel1 := xdsresource.WatchRouteConfig(client, rdsName, rw1)
defer rdsCancel1()
rw2 := newRouteConfigWatcher()
rdsCancel2 := xdsresource.WatchRouteConfig(client, rdsName, rw2)
defer rdsCancel2()
// Register the third watch for a different route configuration resource.
rdsNameNewStyle := makeNewStyleRDSName(authority)
rw3 := newRouteConfigWatcher()
rdsCancel3 := xdsresource.WatchRouteConfig(client, rdsNameNewStyle, rw3)
defer rdsCancel3()
// Configure the management server to return two route configuration
// resources, corresponding to the registered watches.
resources := e2e.UpdateOptions{
NodeID: nodeID,
Routes: []*v3routepb.RouteConfiguration{
e2e.DefaultRouteConfig(rdsName, ldsName, cdsName),
e2e.DefaultRouteConfig(rdsNameNewStyle, ldsName, cdsName),
},
SkipValidation: true,
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
// Verify the contents of the received update for the all watchers. The two
// resources returned differ only in the resource name. Therefore the
// expected update is the same for all the watchers.
wantUpdate := routeConfigUpdateErrTuple{
update: xdsresource.RouteConfigUpdate{
VirtualHosts: []*xdsresource.VirtualHost{
{
Domains: []string{ldsName},
Routes: []*xdsresource.Route{
{
Prefix: newStringP("/"),
ActionType: xdsresource.RouteActionRoute,
WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}},
},
},
},
},
},
}
if err := verifyRouteConfigUpdate(ctx, rw1.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
if err := verifyRouteConfigUpdate(ctx, rw2.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
if err := verifyRouteConfigUpdate(ctx, rw3.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
}
// TestRDSWatch_ResourceCaching covers the case where a watch is registered for
// a resource which is already present in the cache. The test verifies that the
// watch callback is invoked with the contents from the cache, instead of a
// request being sent to the management server.
func (s) TestRDSWatch_ResourceCaching(t *testing.T) {
firstRequestReceived := false
firstAckReceived := grpcsync.NewEvent()
secondRequestReceived := grpcsync.NewEvent()
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
// The first request has an empty version string.
if !firstRequestReceived && req.GetVersionInfo() == "" {
firstRequestReceived = true
return nil
}
// The first ack has a non-empty version string.
if !firstAckReceived.HasFired() && req.GetVersionInfo() != "" {
firstAckReceived.Fire()
return nil
}
// Any requests after the first request and ack, are not expected.
secondRequestReceived.Fire()
return nil
},
})
nodeID := uuid.New().String()
bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
// Create an xDS client with the above bootstrap contents.
config, err := bootstrap.NewConfigFromContents(bc)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
}
pool := xdsclient.NewPool(config)
client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: t.Name(),
})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
defer close()
// Register a watch for a route configuration resource and have the watch
// callback push the received update on to a channel.
rw1 := newRouteConfigWatcher()
rdsCancel1 := xdsresource.WatchRouteConfig(client, rdsName, rw1)
defer rdsCancel1()
// Configure the management server to return a single route configuration
// resource, corresponding to the one we registered a watch for.
resources := e2e.UpdateOptions{
NodeID: nodeID,
Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, cdsName)},
SkipValidation: true,
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
// Verify the contents of the received update.
wantUpdate := routeConfigUpdateErrTuple{
update: xdsresource.RouteConfigUpdate{
VirtualHosts: []*xdsresource.VirtualHost{
{
Domains: []string{ldsName},
Routes: []*xdsresource.Route{
{
Prefix: newStringP("/"),
ActionType: xdsresource.RouteActionRoute,
WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}},
},
},
},
},
},
}
if err := verifyRouteConfigUpdate(ctx, rw1.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
select {
case <-ctx.Done():
t.Fatal("timeout when waiting for receipt of ACK at the management server")
case <-firstAckReceived.Done():
}
// Register another watch for the same resource. This should get the update
// from the cache.
rw2 := newRouteConfigWatcher()
rdsCancel2 := xdsresource.WatchRouteConfig(client, rdsName, rw2)
defer rdsCancel2()
if err := verifyRouteConfigUpdate(ctx, rw2.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
// No request should get sent out as part of this watch.
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
defer sCancel()
select {
case <-sCtx.Done():
case <-secondRequestReceived.Done():
t.Fatal("xdsClient sent out request instead of using update from cache")
}
}
// TestRDSWatch_ExpiryTimerFiresBeforeResponse tests the case where the client
// does not receive an RDS response for the request that it sends. The test
// verifies that the watch callback is invoked with an error once the
// watchExpiryTimer fires.
func (s) TestRDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) {
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
nodeID := uuid.New().String()
bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
// Create an xDS client talking to the above management server.
config, err := bootstrap.NewConfigFromContents(bc)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
}
pool := xdsclient.NewPool(config)
client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: t.Name(),
WatchExpiryTimeout: defaultTestWatchExpiryTimeout,
})
if err != nil {
t.Fatalf("Failed to create an xDS client: %v", err)
}
defer close()
// Register a watch for a resource which is expected to fail with an error
// after the watch expiry timer fires.
rw := newRouteConfigWatcher()
rdsCancel := xdsresource.WatchRouteConfig(client, rdsName, rw)
defer rdsCancel()
// Wait for the watch expiry timer to fire.
<-time.After(defaultTestWatchExpiryTimeout)
// Verify that an empty update with the expected error is received.
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
wantErr := xdsresource.NewError(xdsresource.ErrorTypeResourceNotFound, "")
if err := verifyRouteConfigUpdate(ctx, rw.updateCh, routeConfigUpdateErrTuple{err: wantErr}); err != nil {
t.Fatal(err)
}
}
// TestRDSWatch_ValidResponseCancelsExpiryTimerBehavior tests the case where the
// client receives a valid RDS response for the request that it sends. The test
// verifies that the behavior associated with the expiry timer (i.e, callback
// invocation with error) does not take place.
func (s) TestRDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) {
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
nodeID := uuid.New().String()
bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
// Create an xDS client talking to the above management server.
config, err := bootstrap.NewConfigFromContents(bc)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
}
pool := xdsclient.NewPool(config)
client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: t.Name(),
WatchExpiryTimeout: defaultTestWatchExpiryTimeout,
})
if err != nil {
t.Fatalf("Failed to create an xDS client: %v", err)
}
defer close()
// Register a watch for a route configuration resource and have the watch
// callback push the received update on to a channel.
rw := newRouteConfigWatcher()
rdsCancel := xdsresource.WatchRouteConfig(client, rdsName, rw)
defer rdsCancel()
// Configure the management server to return a single route configuration
// resource, corresponding to the one we registered a watch for.
resources := e2e.UpdateOptions{
NodeID: nodeID,
Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, cdsName)},
SkipValidation: true,
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
// Verify the contents of the received update.
wantUpdate := routeConfigUpdateErrTuple{
update: xdsresource.RouteConfigUpdate{
VirtualHosts: []*xdsresource.VirtualHost{
{
Domains: []string{ldsName},
Routes: []*xdsresource.Route{
{
Prefix: newStringP("/"),
ActionType: xdsresource.RouteActionRoute,
WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}},
},
},
},
},
},
}
if err := verifyRouteConfigUpdate(ctx, rw.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
// Wait for the watch expiry timer to fire, and verify that the callback is
// not invoked.
<-time.After(defaultTestWatchExpiryTimeout)
if err := verifyNoRouteConfigUpdate(ctx, rw.updateCh); err != nil {
t.Fatal(err)
}
}
// TestRDSWatch_NACKError covers the case where an update from the management
// server is NACK'ed by the xdsclient. The test verifies that the error is
// propagated to the watcher.
func (s) TestRDSWatch_NACKError(t *testing.T) {
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
nodeID := uuid.New().String()
bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
// Create an xDS client with the above bootstrap contents.
config, err := bootstrap.NewConfigFromContents(bc)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
}
pool := xdsclient.NewPool(config)
client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: t.Name(),
})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
defer close()
// Register a watch for a route configuration resource and have the watch
// callback push the received update on to a channel.
rw := newRouteConfigWatcher()
rdsCancel := xdsresource.WatchRouteConfig(client, rdsName, rw)
defer rdsCancel()
// Configure the management server to return a single route configuration
// resource which is expected to be NACKed by the client.
resources := e2e.UpdateOptions{
NodeID: nodeID,
Routes: []*v3routepb.RouteConfiguration{badRouteConfigResource(rdsName, ldsName, cdsName)},
SkipValidation: true,
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
// Verify that the expected error is propagated to the watcher.
u, err := rw.updateCh.Receive(ctx)
if err != nil {
t.Fatalf("timeout when waiting for a route configuration resource from the management server: %v", err)
}
gotErr := u.(routeConfigUpdateErrTuple).err
if gotErr == nil || !strings.Contains(gotErr.Error(), wantRouteConfigNACKErr) {
t.Fatalf("update received with error: %v, want %q", gotErr, wantRouteConfigNACKErr)
}
}
// TestRDSWatch_PartialValid covers the case where a response from the
// management server contains both valid and invalid resources and is expected
// to be NACK'ed by the xdsclient. The test verifies that watchers corresponding
// to the valid resource receive the update, while watchers corresponding to the
// invalid resource receive an error.
func (s) TestRDSWatch_PartialValid(t *testing.T) {
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
nodeID := uuid.New().String()
authority := makeAuthorityName(t.Name())
bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
Servers: []byte(fmt.Sprintf(`[{
"server_uri": %q,
"channel_creds": [{"type": "insecure"}]
}]`, mgmtServer.Address)),
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
Authorities: map[string]json.RawMessage{
// Xdstp style resource names used in this test use a slash removed
// version of t.Name as their authority, and the empty config
// results in the top-level xds server configuration being used for
// this authority.
authority: []byte(`{}`),
},
})
if err != nil {
t.Fatalf("Failed to create bootstrap configuration: %v", err)
}
// Create an xDS client with the above bootstrap contents.
config, err := bootstrap.NewConfigFromContents(bc)
if err != nil {
t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
}
pool := xdsclient.NewPool(config)
client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
Name: t.Name(),
})
if err != nil {
t.Fatalf("Failed to create xDS client: %v", err)
}
defer close()
// Register two watches for route configuration resources. The first watch
// is expected to receive an error because the received resource is NACKed.
// The second watch is expected to get a good update.
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
badResourceName := rdsName
rw1 := newRouteConfigWatcher()
rdsCancel1 := xdsresource.WatchRouteConfig(client, badResourceName, rw1)
defer rdsCancel1()
goodResourceName := makeNewStyleRDSName(authority)
rw2 := newRouteConfigWatcher()
rdsCancel2 := xdsresource.WatchRouteConfig(client, goodResourceName, rw2)
defer rdsCancel2()
// Configure the management server to return two route configuration
// resources, corresponding to the registered watches.
resources := e2e.UpdateOptions{
NodeID: nodeID,
Routes: []*v3routepb.RouteConfiguration{
badRouteConfigResource(badResourceName, ldsName, cdsName),
e2e.DefaultRouteConfig(goodResourceName, ldsName, cdsName),
},
SkipValidation: true,
}
if err := mgmtServer.Update(ctx, resources); err != nil {
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
}
// Verify that the expected error is propagated to the watcher which
// requested for the bad resource.
u, err := rw1.updateCh.Receive(ctx)
if err != nil {
t.Fatalf("timeout when waiting for a route configuration resource from the management server: %v", err)
}
gotErr := u.(routeConfigUpdateErrTuple).err
if gotErr == nil || !strings.Contains(gotErr.Error(), wantRouteConfigNACKErr) {
t.Fatalf("update received with error: %v, want %q", gotErr, wantRouteConfigNACKErr)
}
// Verify that the watcher watching the good resource receives a good
// update.
wantUpdate := routeConfigUpdateErrTuple{
update: xdsresource.RouteConfigUpdate{
VirtualHosts: []*xdsresource.VirtualHost{
{
Domains: []string{ldsName},
Routes: []*xdsresource.Route{
{
Prefix: newStringP("/"),
ActionType: xdsresource.RouteActionRoute,
WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}},
},
},
},
},
},
}
if err := verifyRouteConfigUpdate(ctx, rw2.updateCh, wantUpdate); err != nil {
t.Fatal(err)
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,33 @@
/*
*
* Copyright 2020 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsclient_test
import (
"testing"
"google.golang.org/grpc/internal/grpctest"
)
type s struct {
grpctest.Tester
}
func Test(t *testing.T) {
grpctest.RunSubTests(t, s{})
}

View File

@@ -0,0 +1,252 @@
/*
*
* Copyright 2023 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
// Package converter provides converters to convert proto load balancing
// configuration, defined by the xDS API spec, to JSON load balancing
// configuration. These converters are registered by proto type in a registry,
// which gets pulled from based off proto type passed in.
package converter
import (
"encoding/json"
"fmt"
"strings"
"google.golang.org/grpc/balancer"
"google.golang.org/grpc/balancer/leastrequest"
"google.golang.org/grpc/balancer/pickfirst"
"google.golang.org/grpc/balancer/ringhash"
"google.golang.org/grpc/balancer/roundrobin"
"google.golang.org/grpc/balancer/weightedroundrobin"
iringhash "google.golang.org/grpc/internal/ringhash"
internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
"google.golang.org/grpc/internal/xds/balancer/wrrlocality"
"google.golang.org/grpc/internal/xds/xdsclient/xdslbregistry"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/structpb"
v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1"
v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
v3clientsideweightedroundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3"
v3leastrequestpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/least_request/v3"
v3pickfirstpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/pick_first/v3"
v3ringhashpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/ring_hash/v3"
v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3"
)
func init() {
xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.client_side_weighted_round_robin.v3.ClientSideWeightedRoundRobin", convertWeightedRoundRobinProtoToServiceConfig)
xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.ring_hash.v3.RingHash", convertRingHashProtoToServiceConfig)
xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.pick_first.v3.PickFirst", convertPickFirstProtoToServiceConfig)
xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.round_robin.v3.RoundRobin", convertRoundRobinProtoToServiceConfig)
xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.wrr_locality.v3.WrrLocality", convertWRRLocalityProtoToServiceConfig)
xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest", convertLeastRequestProtoToServiceConfig)
xdslbregistry.Register("type.googleapis.com/udpa.type.v1.TypedStruct", convertV1TypedStructToServiceConfig)
xdslbregistry.Register("type.googleapis.com/xds.type.v3.TypedStruct", convertV3TypedStructToServiceConfig)
}
const (
defaultRingHashMinSize = 1024
defaultRingHashMaxSize = 8 * 1024 * 1024 // 8M
defaultLeastRequestChoiceCount = 2
)
func convertRingHashProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) {
rhProto := &v3ringhashpb.RingHash{}
if err := proto.Unmarshal(rawProto, rhProto); err != nil {
return nil, fmt.Errorf("failed to unmarshal resource: %v", err)
}
if rhProto.GetHashFunction() != v3ringhashpb.RingHash_XX_HASH {
return nil, fmt.Errorf("unsupported ring_hash hash function %v", rhProto.GetHashFunction())
}
var minSize, maxSize uint64 = defaultRingHashMinSize, defaultRingHashMaxSize
if min := rhProto.GetMinimumRingSize(); min != nil {
minSize = min.GetValue()
}
if max := rhProto.GetMaximumRingSize(); max != nil {
maxSize = max.GetValue()
}
rhCfg := &iringhash.LBConfig{
MinRingSize: minSize,
MaxRingSize: maxSize,
}
rhCfgJSON, err := json.Marshal(rhCfg)
if err != nil {
return nil, fmt.Errorf("error marshaling JSON for type %T: %v", rhCfg, err)
}
return makeBalancerConfigJSON(ringhash.Name, rhCfgJSON), nil
}
type pfConfig struct {
ShuffleAddressList bool `json:"shuffleAddressList"`
}
func convertPickFirstProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) {
pfProto := &v3pickfirstpb.PickFirst{}
if err := proto.Unmarshal(rawProto, pfProto); err != nil {
return nil, fmt.Errorf("failed to unmarshal resource: %v", err)
}
pfCfg := &pfConfig{ShuffleAddressList: pfProto.GetShuffleAddressList()}
js, err := json.Marshal(pfCfg)
if err != nil {
return nil, fmt.Errorf("error marshaling JSON for type %T: %v", pfCfg, err)
}
return makeBalancerConfigJSON(pickfirst.Name, js), nil
}
func convertRoundRobinProtoToServiceConfig([]byte, int) (json.RawMessage, error) {
return makeBalancerConfigJSON(roundrobin.Name, json.RawMessage("{}")), nil
}
type wrrLocalityLBConfig struct {
ChildPolicy json.RawMessage `json:"childPolicy,omitempty"`
}
func convertWRRLocalityProtoToServiceConfig(rawProto []byte, depth int) (json.RawMessage, error) {
wrrlProto := &v3wrrlocalitypb.WrrLocality{}
if err := proto.Unmarshal(rawProto, wrrlProto); err != nil {
return nil, fmt.Errorf("failed to unmarshal resource: %v", err)
}
epJSON, err := xdslbregistry.ConvertToServiceConfig(wrrlProto.GetEndpointPickingPolicy(), depth+1)
if err != nil {
return nil, fmt.Errorf("error converting endpoint picking policy: %v for %+v", err, wrrlProto)
}
wrrLCfg := wrrLocalityLBConfig{
ChildPolicy: epJSON,
}
lbCfgJSON, err := json.Marshal(wrrLCfg)
if err != nil {
return nil, fmt.Errorf("error marshaling JSON for type %T: %v", wrrLCfg, err)
}
return makeBalancerConfigJSON(wrrlocality.Name, lbCfgJSON), nil
}
func convertWeightedRoundRobinProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) {
cswrrProto := &v3clientsideweightedroundrobinpb.ClientSideWeightedRoundRobin{}
if err := proto.Unmarshal(rawProto, cswrrProto); err != nil {
return nil, fmt.Errorf("failed to unmarshal resource: %v", err)
}
wrrLBCfg := &wrrLBConfig{}
// Only set fields if specified in proto. If not set, ParseConfig of the WRR
// will populate the config with defaults.
if enableOOBLoadReportCfg := cswrrProto.GetEnableOobLoadReport(); enableOOBLoadReportCfg != nil {
wrrLBCfg.EnableOOBLoadReport = enableOOBLoadReportCfg.GetValue()
}
if oobReportingPeriodCfg := cswrrProto.GetOobReportingPeriod(); oobReportingPeriodCfg != nil {
wrrLBCfg.OOBReportingPeriod = internalserviceconfig.Duration(oobReportingPeriodCfg.AsDuration())
}
if blackoutPeriodCfg := cswrrProto.GetBlackoutPeriod(); blackoutPeriodCfg != nil {
wrrLBCfg.BlackoutPeriod = internalserviceconfig.Duration(blackoutPeriodCfg.AsDuration())
}
if weightExpirationPeriodCfg := cswrrProto.GetWeightExpirationPeriod(); weightExpirationPeriodCfg != nil {
wrrLBCfg.WeightExpirationPeriod = internalserviceconfig.Duration(weightExpirationPeriodCfg.AsDuration())
}
if weightUpdatePeriodCfg := cswrrProto.GetWeightUpdatePeriod(); weightUpdatePeriodCfg != nil {
wrrLBCfg.WeightUpdatePeriod = internalserviceconfig.Duration(weightUpdatePeriodCfg.AsDuration())
}
if errorUtilizationPenaltyCfg := cswrrProto.GetErrorUtilizationPenalty(); errorUtilizationPenaltyCfg != nil {
wrrLBCfg.ErrorUtilizationPenalty = float64(errorUtilizationPenaltyCfg.GetValue())
}
lbCfgJSON, err := json.Marshal(wrrLBCfg)
if err != nil {
return nil, fmt.Errorf("error marshaling JSON for type %T: %v", wrrLBCfg, err)
}
return makeBalancerConfigJSON(weightedroundrobin.Name, lbCfgJSON), nil
}
func convertLeastRequestProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) {
lrProto := &v3leastrequestpb.LeastRequest{}
if err := proto.Unmarshal(rawProto, lrProto); err != nil {
return nil, fmt.Errorf("failed to unmarshal resource: %v", err)
}
// "The configuration for the Least Request LB policy is the
// least_request_lb_config field. The field is optional; if not present,
// defaults will be assumed for all of its values." - A48
choiceCount := uint32(defaultLeastRequestChoiceCount)
if cc := lrProto.GetChoiceCount(); cc != nil {
choiceCount = cc.GetValue()
}
lrCfg := &leastrequest.LBConfig{ChoiceCount: choiceCount}
js, err := json.Marshal(lrCfg)
if err != nil {
return nil, fmt.Errorf("error marshaling JSON for type %T: %v", lrCfg, err)
}
return makeBalancerConfigJSON(leastrequest.Name, js), nil
}
func convertV1TypedStructToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) {
tsProto := &v1xdsudpatypepb.TypedStruct{}
if err := proto.Unmarshal(rawProto, tsProto); err != nil {
return nil, fmt.Errorf("failed to unmarshal resource: %v", err)
}
return convertCustomPolicy(tsProto.GetTypeUrl(), tsProto.GetValue())
}
func convertV3TypedStructToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) {
tsProto := &v3xdsxdstypepb.TypedStruct{}
if err := proto.Unmarshal(rawProto, tsProto); err != nil {
return nil, fmt.Errorf("failed to unmarshal resource: %v", err)
}
return convertCustomPolicy(tsProto.GetTypeUrl(), tsProto.GetValue())
}
// convertCustomPolicy attempts to prepare json configuration for a custom lb
// proto, which specifies the gRPC balancer type and configuration. Returns the
// converted json and an error which should cause caller to error if error
// converting. If both json and error returned are nil, it means the gRPC
// Balancer registry does not contain that balancer type, and the caller should
// continue to the next policy.
func convertCustomPolicy(typeURL string, s *structpb.Struct) (json.RawMessage, error) {
// The gRPC policy name will be the "type name" part of the value of the
// type_url field in the TypedStruct. We get this by using the part after
// the last / character. Can assume a valid type_url from the control plane.
pos := strings.LastIndex(typeURL, "/")
name := typeURL[pos+1:]
if balancer.Get(name) == nil {
return nil, nil
}
rawJSON, err := json.Marshal(s)
if err != nil {
return nil, fmt.Errorf("error converting custom lb policy %v: %v for %+v", err, typeURL, s)
}
// The Struct contained in the TypedStruct will be returned as-is as the
// configuration JSON object.
return makeBalancerConfigJSON(name, rawJSON), nil
}
type wrrLBConfig struct {
EnableOOBLoadReport bool `json:"enableOobLoadReport,omitempty"`
OOBReportingPeriod internalserviceconfig.Duration `json:"oobReportingPeriod,omitempty"`
BlackoutPeriod internalserviceconfig.Duration `json:"blackoutPeriod,omitempty"`
WeightExpirationPeriod internalserviceconfig.Duration `json:"weightExpirationPeriod,omitempty"`
WeightUpdatePeriod internalserviceconfig.Duration `json:"weightUpdatePeriod,omitempty"`
ErrorUtilizationPenalty float64 `json:"errorUtilizationPenalty,omitempty"`
}
func makeBalancerConfigJSON(name string, value json.RawMessage) []byte {
return []byte(fmt.Sprintf(`[{%q: %s}]`, name, value))
}

View File

@@ -0,0 +1,82 @@
/*
*
* Copyright 2023 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
// Package xdslbregistry provides a registry of converters that convert proto
// from load balancing configuration, defined by the xDS API spec, to JSON load
// balancing configuration.
package xdslbregistry
import (
"encoding/json"
"fmt"
v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
)
var (
// m is a map from proto type to Converter.
m = make(map[string]Converter)
)
// Register registers the converter to the map keyed on a proto type. Must be
// called at init time. Not thread safe.
func Register(protoType string, c Converter) {
m[protoType] = c
}
// SetRegistry sets the xDS LB registry. Must be called at init time. Not thread
// safe.
func SetRegistry(registry map[string]Converter) {
m = registry
}
// Converter converts raw proto bytes into the internal Go JSON representation
// of the proto passed. Returns the json message, and an error. If both
// returned are nil, it represents continuing to the next proto.
type Converter func([]byte, int) (json.RawMessage, error)
// ConvertToServiceConfig converts a proto Load Balancing Policy configuration
// into a json string. Returns an error if:
// - no supported policy found
// - there is more than 16 layers of recursion in the configuration
// - a failure occurs when converting the policy
func ConvertToServiceConfig(lbPolicy *v3clusterpb.LoadBalancingPolicy, depth int) (json.RawMessage, error) {
// "Configurations that require more than 16 levels of recursion are
// considered invalid and should result in a NACK response." - A51
if depth > 15 {
return nil, fmt.Errorf("lb policy %v exceeds max depth supported: 16 layers", lbPolicy)
}
// "This function iterate over the list of policy messages in
// LoadBalancingPolicy, attempting to convert each one to gRPC form,
// stopping at the first supported policy." - A52
for _, policy := range lbPolicy.GetPolicies() {
converter := m[policy.GetTypedExtensionConfig().GetTypedConfig().GetTypeUrl()]
// "Any entry not in the above list is unsupported and will be skipped."
// - A52
if converter == nil {
continue
}
json, err := converter(policy.GetTypedExtensionConfig().GetTypedConfig().GetValue(), depth)
if json == nil && err == nil {
continue
}
return json, err
}
return nil, fmt.Errorf("no supported policy found in policy list +%v", lbPolicy)
}

View File

@@ -0,0 +1,459 @@
/*
*
* Copyright 2023 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
// Package xdslbregistry_test contains test cases for the xDS LB Policy Registry.
package xdslbregistry_test
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
_ "google.golang.org/grpc/balancer/roundrobin"
"google.golang.org/grpc/internal/balancer/stub"
"google.golang.org/grpc/internal/grpctest"
"google.golang.org/grpc/internal/pretty"
internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/xds/balancer/wrrlocality"
"google.golang.org/grpc/internal/xds/xdsclient/xdslbregistry"
_ "google.golang.org/grpc/xds" // Register the xDS LB Registry Converters.
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/structpb"
"google.golang.org/protobuf/types/known/wrapperspb"
v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1"
v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
v3clientsideweightedroundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3"
v3leastrequestpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/least_request/v3"
v3maglevpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/maglev/v3"
v3pickfirstpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/pick_first/v3"
v3ringhashpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/ring_hash/v3"
v3roundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/round_robin/v3"
v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3"
)
type s struct {
grpctest.Tester
}
func Test(t *testing.T) {
grpctest.RunSubTests(t, s{})
}
func wrrLocalityBalancerConfig(childPolicy *internalserviceconfig.BalancerConfig) *internalserviceconfig.BalancerConfig {
return &internalserviceconfig.BalancerConfig{
Name: wrrlocality.Name,
Config: &wrrlocality.LBConfig{
ChildPolicy: childPolicy,
},
}
}
func (s) TestConvertToServiceConfigSuccess(t *testing.T) {
const customLBPolicyName = "myorg.MyCustomLeastRequestPolicy"
stub.Register(customLBPolicyName, stub.BalancerFuncs{})
tests := []struct {
name string
policy *v3clusterpb.LoadBalancingPolicy
wantConfig string // JSON config
lrEnabled bool
}{
{
name: "ring_hash",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v3ringhashpb.RingHash{
HashFunction: v3ringhashpb.RingHash_XX_HASH,
MinimumRingSize: wrapperspb.UInt64(10),
MaximumRingSize: wrapperspb.UInt64(100),
}),
},
},
},
},
wantConfig: `[{"ring_hash_experimental": { "minRingSize": 10, "maxRingSize": 100 }}]`,
},
{
name: "least_request",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v3leastrequestpb.LeastRequest{
ChoiceCount: wrapperspb.UInt32(3),
}),
},
},
},
},
wantConfig: `[{"least_request_experimental": { "choiceCount": 3 }}]`,
lrEnabled: true,
},
{
name: "pick_first_shuffle",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v3pickfirstpb.PickFirst{
ShuffleAddressList: true,
}),
},
},
},
},
wantConfig: `[{"pick_first": { "shuffleAddressList": true }}]`,
},
{
name: "pick_first",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v3pickfirstpb.PickFirst{}),
},
},
},
},
wantConfig: `[{"pick_first": { "shuffleAddressList": false }}]`,
},
{
name: "round_robin",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v3roundrobinpb.RoundRobin{}),
},
},
},
},
wantConfig: `[{"round_robin": {}}]`,
},
{
name: "weighted_round_robin",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v3clientsideweightedroundrobinpb.ClientSideWeightedRoundRobin{}),
},
},
},
},
wantConfig: `[{"weighted_round_robin": {}}]`,
},
{
name: "weighted_round_robin_populated",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v3clientsideweightedroundrobinpb.ClientSideWeightedRoundRobin{
EnableOobLoadReport: wrapperspb.Bool(true),
OobReportingPeriod: durationpb.New(10 * time.Second),
BlackoutPeriod: durationpb.New(5 * time.Second),
WeightExpirationPeriod: durationpb.New(3 * time.Minute),
WeightUpdatePeriod: durationpb.New(1 * time.Second),
ErrorUtilizationPenalty: wrapperspb.Float(1.5),
}),
},
},
},
},
wantConfig: `[{"weighted_round_robin": {
"enableOobLoadReport": true,
"oobReportingPeriod": "10s",
"blackoutPeriod": "5s",
"weightExpirationPeriod": "180s",
"weightUpdatePeriod": "1s",
"errorUtilizationPenalty": 1.5
}}]`,
},
{
name: "round_robin_ring_hash_use_first_supported",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v3roundrobinpb.RoundRobin{}),
},
},
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v3ringhashpb.RingHash{
HashFunction: v3ringhashpb.RingHash_XX_HASH,
MinimumRingSize: wrapperspb.UInt64(10),
MaximumRingSize: wrapperspb.UInt64(100),
}),
},
},
},
},
wantConfig: `[{"round_robin": {}}]`,
},
{
name: "pf_rr_use_pick_first",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v3pickfirstpb.PickFirst{
ShuffleAddressList: true,
}),
},
},
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v3roundrobinpb.RoundRobin{}),
},
},
},
},
wantConfig: `[{"pick_first": { "shuffleAddressList": true }}]`,
},
{
name: "custom_lb_type_v3_struct",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
// The type not registered in gRPC Policy registry.
// Should fallback to next policy in list.
TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
TypeUrl: "type.googleapis.com/myorg.ThisTypeDoesNotExist",
Value: &structpb.Struct{},
}),
},
},
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
TypeUrl: "type.googleapis.com/myorg.MyCustomLeastRequestPolicy",
Value: &structpb.Struct{},
}),
},
},
},
},
wantConfig: `[{"myorg.MyCustomLeastRequestPolicy": {}}]`,
},
{
name: "custom_lb_type_v1_struct",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v1xdsudpatypepb.TypedStruct{
TypeUrl: "type.googleapis.com/myorg.MyCustomLeastRequestPolicy",
Value: &structpb.Struct{},
}),
},
},
},
},
wantConfig: `[{"myorg.MyCustomLeastRequestPolicy": {}}]`,
},
{
name: "wrr_locality_child_round_robin",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: wrrLocalityAny(t, &v3roundrobinpb.RoundRobin{}),
},
},
},
},
wantConfig: `[{"xds_wrr_locality_experimental": { "childPolicy": [{"round_robin": {}}] }}]`,
},
{
name: "wrr_locality_child_custom_lb_type_v3_struct",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: wrrLocalityAny(t, &v3xdsxdstypepb.TypedStruct{
TypeUrl: "type.googleapis.com/myorg.MyCustomLeastRequestPolicy",
Value: &structpb.Struct{},
}),
},
},
},
},
wantConfig: `[{"xds_wrr_locality_experimental": { "childPolicy": [{"myorg.MyCustomLeastRequestPolicy": {}}] }}]`,
},
{
name: "on-the-boundary-of-recursive-limit",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: wrrLocalityAny(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, &v3roundrobinpb.RoundRobin{}))))))))))))))),
},
},
},
},
wantConfig: jsonMarshal(t, wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(&internalserviceconfig.BalancerConfig{
Name: "round_robin",
})))))))))))))))),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
rawJSON, err := xdslbregistry.ConvertToServiceConfig(test.policy, 0)
if err != nil {
t.Fatalf("ConvertToServiceConfig(%s) failed: %v", pretty.ToJSON(test.policy), err)
}
// got and want must be unmarshalled since JSON strings shouldn't
// generally be directly compared.
var got []map[string]any
if err := json.Unmarshal(rawJSON, &got); err != nil {
t.Fatalf("Error unmarshalling rawJSON (%q): %v", rawJSON, err)
}
var want []map[string]any
if err := json.Unmarshal(json.RawMessage(test.wantConfig), &want); err != nil {
t.Fatalf("Error unmarshalling wantConfig (%q): %v", test.wantConfig, err)
}
if diff := cmp.Diff(got, want); diff != "" {
t.Fatalf("ConvertToServiceConfig() got unexpected output, diff (-got +want): %v", diff)
}
})
}
}
func jsonMarshal(t *testing.T, x any) string {
t.Helper()
js, err := json.Marshal(x)
if err != nil {
t.Fatalf("Error marshalling to JSON (%+v): %v", x, err)
}
return string(js)
}
// TestConvertToServiceConfigFailure tests failure cases of the xDS LB registry
// of converting proto configuration to JSON configuration.
func (s) TestConvertToServiceConfigFailure(t *testing.T) {
tests := []struct {
name string
policy *v3clusterpb.LoadBalancingPolicy
wantErr string
}{
{
name: "not xx_hash function",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v3ringhashpb.RingHash{
HashFunction: v3ringhashpb.RingHash_MURMUR_HASH_2,
MinimumRingSize: wrapperspb.UInt64(10),
MaximumRingSize: wrapperspb.UInt64(100),
}),
},
},
},
},
wantErr: "unsupported ring_hash hash function",
},
{
name: "no-supported-policy",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
// The type not registered in gRPC Policy registry.
TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
TypeUrl: "type.googleapis.com/myorg.ThisTypeDoesNotExist",
Value: &structpb.Struct{},
}),
},
},
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
// Maglev is not yet supported by gRPC.
TypedConfig: testutils.MarshalAny(t, &v3maglevpb.Maglev{}),
},
},
},
},
wantErr: "no supported policy found in policy list",
},
{
name: "exceeds-boundary-of-recursive-limit-by-1",
policy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: wrrLocalityAny(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, wrrLocality(t, &v3roundrobinpb.RoundRobin{})))))))))))))))),
},
},
},
},
wantErr: "exceeds max depth",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, gotErr := xdslbregistry.ConvertToServiceConfig(test.policy, 0)
// Test the error substring to test the different root causes of
// errors. This is more brittle over time, but it's important to
// test the root cause of the errors emitted from the
// ConvertToServiceConfig function call. Also, this package owns the
// error strings so breakages won't come unexpectedly.
if gotErr == nil || !strings.Contains(gotErr.Error(), test.wantErr) {
t.Fatalf("ConvertToServiceConfig() = %v, wantErr %v", gotErr, test.wantErr)
}
})
}
}
// wrrLocality is a helper that takes a proto message and returns a
// WrrLocalityProto with the proto message marshaled into a proto.Any as a
// child.
func wrrLocality(t *testing.T, m proto.Message) *v3wrrlocalitypb.WrrLocality {
return &v3wrrlocalitypb.WrrLocality{
EndpointPickingPolicy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, m),
},
},
},
},
}
}
// wrrLocalityAny takes a proto message and returns a wrr locality proto
// marshaled as an any with an any child set to the marshaled proto message.
func wrrLocalityAny(t *testing.T, m proto.Message) *anypb.Any {
return testutils.MarshalAny(t, wrrLocality(t, m))
}

View File

@@ -0,0 +1,140 @@
/*
*
* Copyright 2022 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"bytes"
"fmt"
"google.golang.org/grpc/internal/xds/bootstrap"
xdsclient "google.golang.org/grpc/internal/xds/clients/xdsclient"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
)
const (
// ClusterResourceTypeName represents the transport agnostic name for the
// cluster resource.
ClusterResourceTypeName = "ClusterResource"
)
// clusterResourceDecoder is an implementation of the xdsclient.Decoder
// interface for listener resources.
type clusterResourceDecoder struct {
bootstrapConfig *bootstrap.Config
serverConfigs map[xdsclient.ServerConfig]*bootstrap.ServerConfig
}
func (d *clusterResourceDecoder) Decode(resource *xdsclient.AnyProto, opts xdsclient.DecodeOptions) (*xdsclient.DecodeResult, error) {
serverCfg, ok := d.serverConfigs[*opts.ServerConfig]
if !ok {
return nil, fmt.Errorf("no server config found for {%+v}", opts.ServerConfig)
}
name, cluster, err := unmarshalClusterResource(resource.ToAny(), serverCfg)
if name == "" {
// Name is unset only when protobuf deserialization fails.
return nil, err
}
if err != nil {
// Protobuf deserialization succeeded, but resource validation failed.
return &xdsclient.DecodeResult{
Name: name,
Resource: &ClusterResourceData{Resource: ClusterUpdate{}},
}, err
}
// Perform extra validation here.
if err := securityConfigValidator(d.bootstrapConfig, cluster.SecurityCfg); err != nil {
return &xdsclient.DecodeResult{
Name: name,
Resource: &ClusterResourceData{Resource: ClusterUpdate{}},
}, err
}
return &xdsclient.DecodeResult{
Name: name,
Resource: &ClusterResourceData{Resource: cluster},
}, nil
}
// ClusterResourceData wraps the configuration of a Cluster resource as received
// from the management server.
type ClusterResourceData struct {
Resource ClusterUpdate
}
// Equal returns true if other is equal to c.
func (c *ClusterResourceData) Equal(other xdsclient.ResourceData) bool {
if other == nil {
return false
}
return bytes.Equal(c.Bytes(), other.Bytes())
}
// Bytes returns the protobuf serialized bytes of the cluster resource proto.
func (c *ClusterResourceData) Bytes() []byte {
return c.Resource.Raw.GetValue()
}
// ClusterWatcher wraps the callbacks to be invoked for different events
// corresponding to the cluster resource being watched. gRFC A88 contains an
// exhaustive list of what method is invoked under what conditions.
type ClusterWatcher interface {
// ResourceChanged indicates a new version of the resource is available.
ResourceChanged(resource *ClusterUpdate, done func())
// ResourceError indicates an error occurred while trying to fetch or
// decode the associated resource. The previous version of the resource
// should be considered invalid.
ResourceError(err error, done func())
// AmbientError indicates an error occurred after a resource has been
// received that should not modify the use of that resource but may provide
// useful information about the state of the XDSClient for debugging
// purposes. The previous version of the resource should still be
// considered valid.
AmbientError(err error, done func())
}
type delegatingClusterWatcher struct {
watcher ClusterWatcher
}
func (d *delegatingClusterWatcher) ResourceChanged(data xdsclient.ResourceData, onDone func()) {
c := data.(*ClusterResourceData)
d.watcher.ResourceChanged(&c.Resource, onDone)
}
func (d *delegatingClusterWatcher) ResourceError(err error, onDone func()) {
d.watcher.ResourceError(err, onDone)
}
func (d *delegatingClusterWatcher) AmbientError(err error, onDone func()) {
d.watcher.AmbientError(err, onDone)
}
// WatchCluster uses xDS to discover the configuration associated with the
// provided cluster resource name.
func WatchCluster(p Producer, name string, w ClusterWatcher) (cancel func()) {
return p.WatchResource(version.V3ClusterURL, name, &delegatingClusterWatcher{watcher: w})
}
// NewClusterResourceTypeDecoder returns a xdsclient.Decoder that wraps
// the xdsresource.clusterType.
func NewClusterResourceTypeDecoder(bc *bootstrap.Config, gServerCfgMap map[xdsclient.ServerConfig]*bootstrap.ServerConfig) xdsclient.Decoder {
return &clusterResourceDecoder{bootstrapConfig: bc, serverConfigs: gServerCfgMap}
}

View File

@@ -0,0 +1,126 @@
/*
*
* Copyright 2022 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"bytes"
"google.golang.org/grpc/internal/xds/bootstrap"
xdsclient "google.golang.org/grpc/internal/xds/clients/xdsclient"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
)
const (
// EndpointsResourceTypeName represents the transport agnostic name for the
// endpoint resource.
EndpointsResourceTypeName = "EndpointsResource"
)
// endpointsResourceDecoder is an implementation of the xdsclient.Decoder
// interface for endpoints resources.
type endpointsResourceDecoder struct {
bootstrapConfig *bootstrap.Config
}
func (d *endpointsResourceDecoder) Decode(resource *xdsclient.AnyProto, _ xdsclient.DecodeOptions) (*xdsclient.DecodeResult, error) {
name, endpoints, err := unmarshalEndpointsResource(resource.ToAny())
if name == "" {
// Name is unset only when protobuf deserialization fails.
return nil, err
}
if err != nil {
// Protobuf deserialization succeeded, but resource validation failed.
return &xdsclient.DecodeResult{
Name: name,
Resource: &ListenerResourceData{Resource: ListenerUpdate{}},
}, err
}
return &xdsclient.DecodeResult{
Name: name,
Resource: &EndpointsResourceData{Resource: endpoints},
}, nil
}
// EndpointsResourceData is an implementation of the xdsclient.ResourceData
// interface for endpoints resources.
type EndpointsResourceData struct {
Resource EndpointsUpdate
}
// Equal returns true if other is equal to e.
func (e *EndpointsResourceData) Equal(other xdsclient.ResourceData) bool {
if other == nil {
return false
}
return bytes.Equal(e.Bytes(), other.Bytes())
}
// Bytes returns the protobuf serialized bytes of the listener resource proto.
func (e *EndpointsResourceData) Bytes() []byte {
return e.Resource.Raw.GetValue()
}
// EndpointsWatcher wraps the callbacks to be invoked for different
// events corresponding to the endpoints resource being watched. gRFC A88
// contains an exhaustive list of what method is invoked under what conditions.
type EndpointsWatcher interface {
// ResourceChanged indicates a new version of the resource is available.
ResourceChanged(resource *EndpointsUpdate, done func())
// ResourceError indicates an error occurred while trying to fetch or
// decode the associated resource. The previous version of the resource
// should be considered invalid.
ResourceError(err error, done func())
// AmbientError indicates an error occurred after a resource has been
// received that should not modify the use of that resource but may provide
// useful information about the state of the XDSClient for debugging
// purposes. The previous version of the resource should still be
// considered valid.
AmbientError(err error, done func())
}
type delegatingEndpointsWatcher struct {
watcher EndpointsWatcher
}
func (d *delegatingEndpointsWatcher) ResourceChanged(data xdsclient.ResourceData, onDone func()) {
e := data.(*EndpointsResourceData)
d.watcher.ResourceChanged(&e.Resource, onDone)
}
func (d *delegatingEndpointsWatcher) ResourceError(err error, onDone func()) {
d.watcher.ResourceError(err, onDone)
}
func (d *delegatingEndpointsWatcher) AmbientError(err error, onDone func()) {
d.watcher.AmbientError(err, onDone)
}
// WatchEndpoints uses xDS to discover the configuration associated with the
// provided endpoints resource name.
func WatchEndpoints(p Producer, name string, w EndpointsWatcher) (cancel func()) {
return p.WatchResource(version.V3EndpointsURL, name, &delegatingEndpointsWatcher{watcher: w})
}
// NewEndpointsResourceTypeDecoder returns a xdsclient.Decoder that wraps
// the xdsresource.endpointsType.
func NewEndpointsResourceTypeDecoder(bc *bootstrap.Config) xdsclient.Decoder {
return &endpointsResourceDecoder{bootstrapConfig: bc}
}

View File

@@ -0,0 +1,79 @@
/*
*
* Copyright 2020 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsresource
import (
"errors"
"fmt"
)
// ErrorType is the type of the error that the watcher will receive from the xds
// client.
type ErrorType int
const (
// ErrorTypeUnknown indicates the error doesn't have a specific type. It is
// the default value, and is returned if the error is not an xds error.
ErrorTypeUnknown ErrorType = iota
// ErrorTypeConnection indicates a connection error from the gRPC client.
ErrorTypeConnection
// ErrorTypeResourceNotFound indicates a resource is not found from the xds
// response. It's typically returned if the resource is removed in the xds
// server.
ErrorTypeResourceNotFound
// ErrorTypeResourceTypeUnsupported indicates the receipt of a message from
// the management server with resources of an unsupported resource type.
ErrorTypeResourceTypeUnsupported
// ErrTypeStreamFailedAfterRecv indicates an ADS stream error, after
// successful receipt of at least one message from the server.
ErrTypeStreamFailedAfterRecv
// ErrorTypeNACKed indicates that configuration provided by the xDS management
// server was NACKed.
ErrorTypeNACKed
)
type xdsClientError struct {
t ErrorType
desc string
}
func (e *xdsClientError) Error() string {
return e.desc
}
// NewErrorf creates an xDS client error. The callbacks are called with this
// error, to pass additional information about the error.
func NewErrorf(t ErrorType, format string, args ...any) error {
return &xdsClientError{t: t, desc: fmt.Sprintf(format, args...)}
}
// NewError creates an xDS client error. The callbacks are called with this
// error, to pass additional information about the error.
func NewError(t ErrorType, message string) error {
return NewErrorf(t, "%s", message)
}
// ErrType returns the error's type.
func ErrType(err error) ErrorType {
var xe *xdsClientError
if errors.As(err, &xe) {
return xe.t
}
return ErrorTypeUnknown
}

View File

@@ -0,0 +1,180 @@
/*
*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"fmt"
"net/netip"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
)
// NetworkFilterChainMap contains the match configuration for network filter
// chains on the server side. It is a multi-level map structure to facilitate
// efficient matching of incoming connections based on destination IP, source
// {type, IP and port}.
type NetworkFilterChainMap struct {
// DstPrefixes is the list of destination prefix entries to match on.
DstPrefixes []DestinationPrefixEntry
}
// DestinationPrefixEntry contains a destination prefix entry and the associated
// source type matchers.
type DestinationPrefixEntry struct {
// Prefix is the destination IP prefix.
Prefix netip.Prefix
// SourceTypeArr contains the source type matchers. The supported source
// types and their associated indices in the array are:
// - 0: Any: matches connection attempts from any source.
// - 1: SameOrLoopback: matches connection attempts from the same host.
// - 2: External: matches connection attempts from a different host.
SourceTypeArr [3]SourcePrefixes
}
// SourcePrefixes contains a list of source prefix entries to match on.
type SourcePrefixes struct {
// Entries is the list of source prefix entries.
Entries []SourcePrefixEntry
}
// SourcePrefixEntry contains a source prefix entry and the associated source
// port matchers.
type SourcePrefixEntry struct {
// Prefix is the source IP prefix.
Prefix netip.Prefix
// PortMap contains the matchers for source ports.
PortMap map[int]NetworkFilterChainConfig
}
// NetworkFilterChainConfig contains the configuration for a network filter
// chain on the server side. The only support network filter is the HTTP
// connection manager.
type NetworkFilterChainConfig struct {
// SecurityCfg contains transport socket security configuration.
SecurityCfg *SecurityConfig
// HTTPConnMgr contains the HTTP connection manager configuration.
HTTPConnMgr *HTTPConnectionManagerConfig
}
// IsEmpty returns true if the NetworkFilterChainConfig contains no
// configuration.
func (n NetworkFilterChainConfig) IsEmpty() bool {
return n.SecurityCfg == nil && n.HTTPConnMgr == nil
}
func processNetworkFilters(filters []*v3listenerpb.Filter, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (*HTTPConnectionManagerConfig, error) {
hcmConfig := &HTTPConnectionManagerConfig{}
seenNames := make(map[string]bool, len(filters))
seenHCM := false
for _, filter := range filters {
name := filter.GetName()
if name == "" {
return nil, fmt.Errorf("network filters {%+v} is missing name field in filter: {%+v}", filters, filter)
}
if seenNames[name] {
return nil, fmt.Errorf("network filters {%+v} has duplicate filter name %q", filters, name)
}
seenNames[name] = true
// Network filters have a oneof field named `config_type` where we
// only support `TypedConfig` variant.
switch typ := filter.GetConfigType().(type) {
case *v3listenerpb.Filter_TypedConfig:
// The typed_config field has an `anypb.Any` proto which could
// directly contain the serialized bytes of the actual filter
// configuration, or it could be encoded as a `TypedStruct`.
// TODO: Add support for `TypedStruct`.
tc := filter.GetTypedConfig()
// The only network filter that we currently support is the v3
// HttpConnectionManager. So, we can directly check the type_url
// and unmarshal the config.
// TODO: Implement a registry of supported network filters (like
// we have for HTTP filters), when we have to support network
// filters other than HttpConnectionManager.
if tc.GetTypeUrl() != version.V3HTTPConnManagerURL {
return nil, fmt.Errorf("network filters {%+v} has unsupported network filter %q in filter {%+v}", filters, tc.GetTypeUrl(), filter)
}
hcm := &v3httppb.HttpConnectionManager{}
if err := tc.UnmarshalTo(hcm); err != nil {
return nil, fmt.Errorf("network filters {%+v} failed unmarshalling of network filter {%+v}: %v", filters, filter, err)
}
// "Any filters after HttpConnectionManager should be ignored during
// connection processing but still be considered for validity.
// HTTPConnectionManager must have valid http_filters." - A36
filters, err := processHTTPFilters(hcm.GetHttpFilters(), true, bc, sc)
if err != nil {
return nil, fmt.Errorf("network filters {%+v} had invalid server side HTTP Filters {%+v}: %v", filters, hcm.GetHttpFilters(), err)
}
if !seenHCM {
// Validate for RBAC in only the HCM that will be used, since this isn't a logical validation failure,
// it's simply a validation to support RBAC HTTP Filter.
// "HttpConnectionManager.xff_num_trusted_hops must be unset or zero and
// HttpConnectionManager.original_ip_detection_extensions must be empty. If
// either field has an incorrect value, the Listener must be NACKed." - A41
if hcm.XffNumTrustedHops != 0 {
return nil, fmt.Errorf("xff_num_trusted_hops must be unset or zero %+v", hcm)
}
if len(hcm.OriginalIpDetectionExtensions) != 0 {
return nil, fmt.Errorf("original_ip_detection_extensions must be empty %+v", hcm)
}
// TODO: Implement terminal filter logic, as per A36.
hcmConfig.HTTPFilters = filters
seenHCM = true
switch hcm.RouteSpecifier.(type) {
case *v3httppb.HttpConnectionManager_Rds:
if hcm.GetRds().GetConfigSource().GetAds() == nil {
return nil, fmt.Errorf("ConfigSource is not ADS: %+v", hcm)
}
name := hcm.GetRds().GetRouteConfigName()
if name == "" {
return nil, fmt.Errorf("empty route_config_name: %+v", hcm)
}
hcmConfig.RouteConfigName = name
case *v3httppb.HttpConnectionManager_RouteConfig:
// "RouteConfiguration validation logic inherits all
// previous validations made for client-side usage as RDS
// does not distinguish between client-side and
// server-side." - A36
// Can specify v3 here, as will never get to this function
// if v2.
routeU, err := generateRDSUpdateFromRouteConfiguration(hcm.GetRouteConfig(), bc, sc)
if err != nil {
return nil, fmt.Errorf("failed to parse inline RDS resp: %v", err)
}
hcmConfig.InlineRouteConfig = &routeU
case nil:
return nil, fmt.Errorf("no RouteSpecifier: %+v", hcm)
default:
return nil, fmt.Errorf("unsupported type %T for RouteSpecifier", hcm.RouteSpecifier)
}
}
default:
return nil, fmt.Errorf("network filters {%+v} has unsupported config_type %T in filter %s", filters, typ, filter.GetName())
}
}
if !seenHCM {
return nil, fmt.Errorf("network filters {%+v} missing HttpConnectionManager filter", filters)
}
return hcmConfig, nil
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,185 @@
/*
*
* Copyright 2022 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"bytes"
"fmt"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/clients/xdsclient"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
)
// ListenerResourceTypeName is a human friendly name for the listener resource.
const ListenerResourceTypeName = "ListenerResource"
// listenerResourceDecoder is an implementation of the xdsclient.Decoder
// interface for listener resources.
type listenerResourceDecoder struct {
bootstrapConfig *bootstrap.Config
serverConfigs map[xdsclient.ServerConfig]*bootstrap.ServerConfig
}
func (d *listenerResourceDecoder) Decode(resource *xdsclient.AnyProto, opts xdsclient.DecodeOptions) (*xdsclient.DecodeResult, error) {
var serverCfg *bootstrap.ServerConfig
if opts.ServerConfig != nil {
serverCfg = d.serverConfigs[*opts.ServerConfig]
}
name, listener, err := unmarshalListenerResource(resource.ToAny(), d.bootstrapConfig, serverCfg)
if name == "" {
// Name is unset only when protobuf deserialization fails.
return nil, err
}
if err != nil {
// Protobuf deserialization succeeded, but resource validation failed.
return &xdsclient.DecodeResult{
Name: name,
Resource: &ListenerResourceData{Resource: ListenerUpdate{}},
}, err
}
// Perform extra validation here.
if err := listenerValidator(d.bootstrapConfig, listener); err != nil {
return &xdsclient.DecodeResult{
Name: name,
Resource: &ListenerResourceData{Resource: ListenerUpdate{}},
}, err
}
return &xdsclient.DecodeResult{
Name: name,
Resource: &ListenerResourceData{Resource: listener},
}, nil
}
func securityConfigValidator(bc *bootstrap.Config, sc *SecurityConfig) error {
if sc == nil {
return nil
}
if sc.IdentityInstanceName != "" {
if _, ok := bc.CertProviderConfigs()[sc.IdentityInstanceName]; !ok {
return fmt.Errorf("identity certificate provider instance name %q missing in bootstrap configuration", sc.IdentityInstanceName)
}
}
if sc.RootInstanceName != "" {
if _, ok := bc.CertProviderConfigs()[sc.RootInstanceName]; !ok {
return fmt.Errorf("root certificate provider instance name %q missing in bootstrap configuration", sc.RootInstanceName)
}
}
return nil
}
func listenerValidator(bc *bootstrap.Config, lis ListenerUpdate) error {
// Validate Filter Chains.
validateFC := func(fc *NetworkFilterChainConfig) error {
if fc == nil {
return nil
}
return securityConfigValidator(bc, fc.SecurityCfg)
}
if lis.TCPListener == nil {
return nil
}
if err := validateFC(&lis.TCPListener.DefaultFilterChain); err != nil {
return err
}
for _, dst := range lis.TCPListener.FilterChains.DstPrefixes {
for _, srcType := range dst.SourceTypeArr {
if len(srcType.Entries) == 0 {
continue
}
for _, src := range srcType.Entries {
for _, fc := range src.PortMap {
if err := validateFC(&fc); err != nil {
return err
}
}
}
}
}
return nil
}
// ListenerResourceData is an implementation of the xdsclient.ResourceData
// interface for listener resources.
type ListenerResourceData struct {
Resource ListenerUpdate
}
// Equal returns true if other is equal to l.
func (l *ListenerResourceData) Equal(other xdsclient.ResourceData) bool {
if other == nil {
return false
}
return bytes.Equal(l.Bytes(), other.Bytes())
}
// Bytes returns the protobuf serialized bytes of the listener resource proto.
func (l *ListenerResourceData) Bytes() []byte {
return l.Resource.Raw.GetValue()
}
// ListenerWatcher wraps the callbacks to be invoked for different
// events corresponding to the listener resource being watched. gRFC A88
// contains an exhaustive list of what method is invoked under what conditions.
type ListenerWatcher interface {
// ResourceChanged indicates a new version of the resource is available.
ResourceChanged(resource *ListenerUpdate, done func())
// ResourceError indicates an error occurred while trying to fetch or
// decode the associated resource. The previous version of the resource
// should be considered invalid.
ResourceError(err error, done func())
// AmbientError indicates an error occurred after a resource has been
// received that should not modify the use of that resource but may provide
// useful information about the state of the XDSClient for debugging
// purposes. The previous version of the resource should still be
// considered valid.
AmbientError(err error, done func())
}
type delegatingListenerWatcher struct {
watcher ListenerWatcher
}
func (d *delegatingListenerWatcher) ResourceChanged(data xdsclient.ResourceData, onDone func()) {
l := data.(*ListenerResourceData)
d.watcher.ResourceChanged(&l.Resource, onDone)
}
func (d *delegatingListenerWatcher) ResourceError(err error, onDone func()) {
d.watcher.ResourceError(err, onDone)
}
func (d *delegatingListenerWatcher) AmbientError(err error, onDone func()) {
d.watcher.AmbientError(err, onDone)
}
// WatchListener uses xDS to discover the configuration associated with the
// provided listener resource name.
func WatchListener(p Producer, name string, w ListenerWatcher) (cancel func()) {
return p.WatchResource(version.V3ListenerURL, name, &delegatingListenerWatcher{watcher: w})
}
// NewListenerResourceTypeDecoder returns a xdsclient.Decoder that wraps
// the xdsresource.listenerType.
func NewListenerResourceTypeDecoder(bc *bootstrap.Config, gServerCfgMap map[xdsclient.ServerConfig]*bootstrap.ServerConfig) xdsclient.Decoder {
return &listenerResourceDecoder{bootstrapConfig: bc, serverConfigs: gServerCfgMap}
}

View File

@@ -0,0 +1,28 @@
/*
*
* Copyright 2023 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package xdsresource
import (
"google.golang.org/grpc/grpclog"
internalgrpclog "google.golang.org/grpc/internal/grpclog"
)
const prefix = "[xds-resource] "
var logger = internalgrpclog.NewPrefixLogger(grpclog.Component("xds"), prefix)

View File

@@ -0,0 +1,231 @@
/*
*
* Copyright 2020 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"fmt"
rand "math/rand/v2"
"strings"
"google.golang.org/grpc/internal/xds/matcher"
"google.golang.org/grpc/metadata"
)
// RouteToMatcher converts a route to a Matcher to match incoming RPC's against.
//
// Only expected to be called on a Route that passed validation checks by the
// xDS client.
func RouteToMatcher(r *Route) *CompositeMatcher {
var pm pathMatcher
switch {
case r.Regex != nil:
pm = newPathRegexMatcher(r.Regex)
case r.Path != nil:
pm = newPathExactMatcher(*r.Path, r.CaseInsensitive)
case r.Prefix != nil:
pm = newPathPrefixMatcher(*r.Prefix, r.CaseInsensitive)
default:
panic("illegal route: missing path_matcher")
}
headerMatchers := make([]matcher.HeaderMatcher, 0, len(r.Headers))
for _, h := range r.Headers {
var matcherT matcher.HeaderMatcher
invert := h.InvertMatch != nil && *h.InvertMatch
switch {
case h.RegexMatch != nil:
matcherT = matcher.NewHeaderRegexMatcher(h.Name, h.RegexMatch, invert)
case h.RangeMatch != nil:
matcherT = matcher.NewHeaderRangeMatcher(h.Name, h.RangeMatch.Start, h.RangeMatch.End, invert)
case h.PresentMatch != nil:
matcherT = matcher.NewHeaderPresentMatcher(h.Name, *h.PresentMatch, invert)
case h.StringMatch != nil:
matcherT = matcher.NewHeaderStringMatcher(h.Name, *h.StringMatch, invert)
default:
panic("illegal route: missing header_match_specifier")
}
headerMatchers = append(headerMatchers, matcherT)
}
var fractionMatcher *fractionMatcher
if r.Fraction != nil {
fractionMatcher = newFractionMatcher(*r.Fraction)
}
return newCompositeMatcher(pm, headerMatchers, fractionMatcher)
}
// CompositeMatcher is a matcher that holds onto many matchers and aggregates
// the matching results.
type CompositeMatcher struct {
pm pathMatcher
hms []matcher.HeaderMatcher
fm *fractionMatcher
}
func newCompositeMatcher(pm pathMatcher, hms []matcher.HeaderMatcher, fm *fractionMatcher) *CompositeMatcher {
return &CompositeMatcher{pm: pm, hms: hms, fm: fm}
}
// Match returns true if all matchers return true.
func (a *CompositeMatcher) Match(method string, md metadata.MD) bool {
if a.pm != nil && !a.pm.match(method) {
return false
}
for _, m := range a.hms {
if !m.Match(md) {
return false
}
}
if a.fm != nil && !a.fm.match() {
return false
}
return true
}
func (a *CompositeMatcher) String() string {
var ret string
if a.pm != nil {
ret += a.pm.String()
}
for _, m := range a.hms {
ret += m.String()
}
if a.fm != nil {
ret += a.fm.String()
}
return ret
}
type fractionMatcher struct {
fraction int64 // real fraction is fraction/1,000,000.
}
func newFractionMatcher(fraction uint32) *fractionMatcher {
return &fractionMatcher{fraction: int64(fraction)}
}
// RandInt64n overwrites rand for control in tests.
var RandInt64n = rand.Int64N
func (fm *fractionMatcher) match() bool {
t := RandInt64n(1000000)
return t <= fm.fraction
}
func (fm *fractionMatcher) String() string {
return fmt.Sprintf("fraction:%v", fm.fraction)
}
type domainMatchType int
const (
domainMatchTypeInvalid domainMatchType = iota
domainMatchTypeUniversal
domainMatchTypePrefix
domainMatchTypeSuffix
domainMatchTypeExact
)
// Exact > Suffix > Prefix > Universal > Invalid.
func (t domainMatchType) betterThan(b domainMatchType) bool {
return t > b
}
func matchTypeForDomain(d string) domainMatchType {
if d == "" {
return domainMatchTypeInvalid
}
if d == "*" {
return domainMatchTypeUniversal
}
if strings.HasPrefix(d, "*") {
return domainMatchTypeSuffix
}
if strings.HasSuffix(d, "*") {
return domainMatchTypePrefix
}
if strings.Contains(d, "*") {
return domainMatchTypeInvalid
}
return domainMatchTypeExact
}
func match(domain, host string) (domainMatchType, bool) {
switch typ := matchTypeForDomain(domain); typ {
case domainMatchTypeInvalid:
return typ, false
case domainMatchTypeUniversal:
return typ, true
case domainMatchTypePrefix:
// abc.*
return typ, strings.HasPrefix(host, strings.TrimSuffix(domain, "*"))
case domainMatchTypeSuffix:
// *.123
return typ, strings.HasSuffix(host, strings.TrimPrefix(domain, "*"))
case domainMatchTypeExact:
return typ, domain == host
default:
return domainMatchTypeInvalid, false
}
}
// FindBestMatchingVirtualHost returns the virtual host whose domains field best
// matches host
//
// The domains field support 4 different matching pattern types:
//
// - Exact match
// - Suffix match (e.g. “*ABC”)
// - Prefix match (e.g. “ABC*)
// - Universal match (e.g. “*”)
//
// The best match is defined as:
// - A match is better if it’s matching pattern type is better.
// * Exact match > suffix match > prefix match > universal match.
//
// - If two matches are of the same pattern type, the longer match is
// better.
// * This is to compare the length of the matching pattern, e.g. “*ABCDE” >
// “*ABC”
func FindBestMatchingVirtualHost(host string, vHosts []*VirtualHost) *VirtualHost { // Maybe move this crap to client
var (
matchVh *VirtualHost
matchType = domainMatchTypeInvalid
matchLen int
)
for _, vh := range vHosts {
for _, domain := range vh.Domains {
typ, matched := match(domain, host)
if typ == domainMatchTypeInvalid {
// The rds response is invalid.
return nil
}
if matchType.betterThan(typ) || matchType == typ && matchLen >= len(domain) || !matched {
// The previous match has better type, or the previous match has
// better length, or this domain isn't a match.
continue
}
matchVh = vh
matchType = typ
matchLen = len(domain)
}
}
return matchVh
}

View File

@@ -0,0 +1,100 @@
/*
*
* Copyright 2020 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"regexp"
"strings"
)
type pathMatcher interface {
match(path string) bool
String() string
}
type pathExactMatcher struct {
// fullPath is all upper case if caseInsensitive is true.
fullPath string
caseInsensitive bool
}
func newPathExactMatcher(p string, caseInsensitive bool) *pathExactMatcher {
ret := &pathExactMatcher{
fullPath: p,
caseInsensitive: caseInsensitive,
}
if caseInsensitive {
ret.fullPath = strings.ToUpper(p)
}
return ret
}
func (pem *pathExactMatcher) match(path string) bool {
if pem.caseInsensitive {
return pem.fullPath == strings.ToUpper(path)
}
return pem.fullPath == path
}
func (pem *pathExactMatcher) String() string {
return "pathExact:" + pem.fullPath
}
type pathPrefixMatcher struct {
// prefix is all upper case if caseInsensitive is true.
prefix string
caseInsensitive bool
}
func newPathPrefixMatcher(p string, caseInsensitive bool) *pathPrefixMatcher {
ret := &pathPrefixMatcher{
prefix: p,
caseInsensitive: caseInsensitive,
}
if caseInsensitive {
ret.prefix = strings.ToUpper(p)
}
return ret
}
func (ppm *pathPrefixMatcher) match(path string) bool {
if ppm.caseInsensitive {
return strings.HasPrefix(strings.ToUpper(path), ppm.prefix)
}
return strings.HasPrefix(path, ppm.prefix)
}
func (ppm *pathPrefixMatcher) String() string {
return "pathPrefix:" + ppm.prefix
}
type pathRegexMatcher struct {
re *regexp.Regexp
}
func newPathRegexMatcher(re *regexp.Regexp) *pathRegexMatcher {
return &pathRegexMatcher{re: re}
}
func (prm *pathRegexMatcher) match(path string) bool {
return prm.re.MatchString(path)
}
func (prm *pathRegexMatcher) String() string {
return "pathRegex:" + prm.re.String()
}

View File

@@ -0,0 +1,93 @@
/*
*
* Copyright 2020 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"regexp"
"testing"
)
func (s) TestPathFullMatcherMatch(t *testing.T) {
tests := []struct {
name string
fullPath string
caseInsensitive bool
path string
want bool
}{
{name: "match", fullPath: "/s/m", path: "/s/m", want: true},
{name: "case insensitive match", fullPath: "/s/m", caseInsensitive: true, path: "/S/m", want: true},
{name: "case insensitive match 2", fullPath: "/s/M", caseInsensitive: true, path: "/S/m", want: true},
{name: "not match", fullPath: "/s/m", path: "/a/b", want: false},
{name: "case insensitive not match", fullPath: "/s/m", caseInsensitive: true, path: "/a/b", want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
fpm := newPathExactMatcher(tt.fullPath, tt.caseInsensitive)
if got := fpm.match(tt.path); got != tt.want {
t.Errorf("{%q}.match(%q) = %v, want %v", tt.fullPath, tt.path, got, tt.want)
}
})
}
}
func (s) TestPathPrefixMatcherMatch(t *testing.T) {
tests := []struct {
name string
prefix string
caseInsensitive bool
path string
want bool
}{
{name: "match", prefix: "/s/", path: "/s/m", want: true},
{name: "case insensitive match", prefix: "/s/", caseInsensitive: true, path: "/S/m", want: true},
{name: "case insensitive match 2", prefix: "/S/", caseInsensitive: true, path: "/s/m", want: true},
{name: "not match", prefix: "/s/", path: "/a/b", want: false},
{name: "case insensitive not match", prefix: "/s/", caseInsensitive: true, path: "/a/b", want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
fpm := newPathPrefixMatcher(tt.prefix, tt.caseInsensitive)
if got := fpm.match(tt.path); got != tt.want {
t.Errorf("{%q}.match(%q) = %v, want %v", tt.prefix, tt.path, got, tt.want)
}
})
}
}
func (s) TestPathRegexMatcherMatch(t *testing.T) {
tests := []struct {
name string
regexPath string
path string
want bool
}{
{name: "match", regexPath: "^/s+/m.*$", path: "/sss/me", want: true},
{name: "not match", regexPath: "^/s+/m*$", path: "/sss/b", want: false},
{name: "no match because only part of path matches with regex", regexPath: "^a+$", path: "ab", want: false},
{name: "match because full path matches with regex", regexPath: "^a+$", path: "aa", want: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
fpm := newPathRegexMatcher(regexp.MustCompile(tt.regexPath))
if got := fpm.match(tt.path); got != tt.want {
t.Errorf("{%q}.match(%q) = %v, want %v", tt.regexPath, tt.path, got, tt.want)
}
})
}
}

View File

@@ -0,0 +1,244 @@
/*
*
* Copyright 2020 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"context"
rand "math/rand/v2"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"google.golang.org/grpc/internal/grpcutil"
iresolver "google.golang.org/grpc/internal/resolver"
"google.golang.org/grpc/internal/xds/matcher"
"google.golang.org/grpc/metadata"
"google.golang.org/protobuf/proto"
)
func (s) TestAndMatcherMatch(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
tests := []struct {
name string
pm pathMatcher
hm matcher.HeaderMatcher
info iresolver.RPCInfo
want bool
}{
{
name: "both_match",
pm: newPathExactMatcher("/a/b", false),
hm: matcher.NewHeaderExactMatcher("th", "tv", false),
info: iresolver.RPCInfo{
Method: "/a/b",
Context: metadata.NewOutgoingContext(ctx, metadata.Pairs("th", "tv")),
},
want: true,
},
{
name: "both_match_with_path_case_insensitive",
pm: newPathExactMatcher("/A/B", true),
hm: matcher.NewHeaderExactMatcher("th", "tv", false),
info: iresolver.RPCInfo{
Method: "/a/b",
Context: metadata.NewOutgoingContext(ctx, metadata.Pairs("th", "tv")),
},
want: true,
},
{
name: "only_one_match",
pm: newPathExactMatcher("/a/b", false),
hm: matcher.NewHeaderExactMatcher("th", "tv", false),
info: iresolver.RPCInfo{
Method: "/z/y",
Context: metadata.NewOutgoingContext(ctx, metadata.Pairs("th", "tv")),
},
want: false,
},
{
name: "both_not match",
pm: newPathExactMatcher("/z/y", false),
hm: matcher.NewHeaderExactMatcher("th", "abc", false),
info: iresolver.RPCInfo{
Method: "/a/b",
Context: metadata.NewOutgoingContext(ctx, metadata.Pairs("th", "tv")),
},
want: false,
},
{
name: "fake_header",
pm: newPathPrefixMatcher("/", false),
hm: matcher.NewHeaderExactMatcher("content-type", "fake", false),
info: iresolver.RPCInfo{
Method: "/a/b",
Context: grpcutil.WithExtraMetadata(ctx, metadata.Pairs(
"content-type", "fake",
)),
},
want: true,
},
{
name: "binary_header",
pm: newPathPrefixMatcher("/", false),
hm: matcher.NewHeaderPresentMatcher("t-bin", true, false),
info: iresolver.RPCInfo{
Method: "/a/b",
Context: grpcutil.WithExtraMetadata(
metadata.NewOutgoingContext(ctx, metadata.Pairs("t-bin", "123")), metadata.Pairs(
"content-type", "fake",
)),
},
// Shouldn't match binary header, even though it's in metadata.
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
a := newCompositeMatcher(tt.pm, []matcher.HeaderMatcher{tt.hm}, nil)
md, _ := metadata.FromOutgoingContext(tt.info.Context)
if extraMD, ok := grpcutil.ExtraMetadata(tt.info.Context); ok {
md = metadata.Join(md, extraMD)
for k := range md {
if strings.HasSuffix(k, "-bin") {
delete(md, k)
}
}
}
if got := a.Match(tt.info.Method, md); got != tt.want {
t.Errorf("match() = %v, want %v", got, tt.want)
}
})
}
}
func (s) TestFractionMatcherMatch(t *testing.T) {
const fraction = 500000
fm := newFractionMatcher(fraction)
defer func() {
RandInt64n = rand.Int64N
}()
// rand > fraction, should return false.
RandInt64n = func(int64) int64 {
return fraction + 1
}
if matched := fm.match(); matched {
t.Errorf("match() = %v, want not match", matched)
}
// rand == fraction, should return true.
RandInt64n = func(int64) int64 {
return fraction
}
if matched := fm.match(); !matched {
t.Errorf("match() = %v, want match", matched)
}
// rand < fraction, should return true.
RandInt64n = func(int64) int64 {
return fraction - 1
}
if matched := fm.match(); !matched {
t.Errorf("match() = %v, want match", matched)
}
}
func (s) TestMatchTypeForDomain(t *testing.T) {
tests := []struct {
d string
want domainMatchType
}{
{d: "", want: domainMatchTypeInvalid},
{d: "*", want: domainMatchTypeUniversal},
{d: "bar.*", want: domainMatchTypePrefix},
{d: "*.abc.com", want: domainMatchTypeSuffix},
{d: "foo.bar.com", want: domainMatchTypeExact},
{d: "foo.*.com", want: domainMatchTypeInvalid},
}
for _, tt := range tests {
if got := matchTypeForDomain(tt.d); got != tt.want {
t.Errorf("matchTypeForDomain(%q) = %v, want %v", tt.d, got, tt.want)
}
}
}
func (s) TestMatch(t *testing.T) {
tests := []struct {
name string
domain string
host string
wantTyp domainMatchType
wantMatched bool
}{
{name: "invalid-empty", domain: "", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false},
{name: "invalid", domain: "a.*.b", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false},
{name: "universal", domain: "*", host: "abc.com", wantTyp: domainMatchTypeUniversal, wantMatched: true},
{name: "prefix-match", domain: "abc.*", host: "abc.123", wantTyp: domainMatchTypePrefix, wantMatched: true},
{name: "prefix-no-match", domain: "abc.*", host: "abcd.123", wantTyp: domainMatchTypePrefix, wantMatched: false},
{name: "suffix-match", domain: "*.123", host: "abc.123", wantTyp: domainMatchTypeSuffix, wantMatched: true},
{name: "suffix-no-match", domain: "*.123", host: "abc.1234", wantTyp: domainMatchTypeSuffix, wantMatched: false},
{name: "exact-match", domain: "foo.bar", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: true},
{name: "exact-no-match", domain: "foo.bar.com", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if gotTyp, gotMatched := match(tt.domain, tt.host); gotTyp != tt.wantTyp || gotMatched != tt.wantMatched {
t.Errorf("match() = %v, %v, want %v, %v", gotTyp, gotMatched, tt.wantTyp, tt.wantMatched)
}
})
}
}
func (s) TestFindBestMatchingVirtualHost(t *testing.T) {
var (
oneExactMatch = &VirtualHost{Domains: []string{"foo.bar.com"}}
oneSuffixMatch = &VirtualHost{Domains: []string{"*.bar.com"}}
onePrefixMatch = &VirtualHost{Domains: []string{"foo.bar.*"}}
oneUniversalMatch = &VirtualHost{Domains: []string{"*"}}
longExactMatch = &VirtualHost{Domains: []string{"v2.foo.bar.com"}}
multipleMatch = &VirtualHost{Domains: []string{"pi.foo.bar.com", "314.*", "*.159"}}
vhs = []*VirtualHost{oneExactMatch, oneSuffixMatch, onePrefixMatch, oneUniversalMatch, longExactMatch, multipleMatch}
)
tests := []struct {
name string
host string
vHosts []*VirtualHost
want *VirtualHost
}{
{name: "exact-match", host: "foo.bar.com", vHosts: vhs, want: oneExactMatch},
{name: "suffix-match", host: "123.bar.com", vHosts: vhs, want: oneSuffixMatch},
{name: "prefix-match", host: "foo.bar.org", vHosts: vhs, want: onePrefixMatch},
{name: "universal-match", host: "abc.123", vHosts: vhs, want: oneUniversalMatch},
{name: "long-exact-match", host: "v2.foo.bar.com", vHosts: vhs, want: longExactMatch},
// Matches suffix "*.bar.com" and exact "pi.foo.bar.com". Takes exact.
{name: "multiple-match-exact", host: "pi.foo.bar.com", vHosts: vhs, want: multipleMatch},
// Matches suffix "*.159" and prefix "foo.bar.*". Takes suffix.
{name: "multiple-match-suffix", host: "foo.bar.159", vHosts: vhs, want: multipleMatch},
// Matches suffix "*.bar.com" and prefix "314.*". Takes suffix.
{name: "multiple-match-prefix", host: "314.bar.com", vHosts: vhs, want: oneSuffixMatch},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := FindBestMatchingVirtualHost(tt.host, tt.vHosts); !cmp.Equal(got, tt.want, cmp.Comparer(proto.Equal)) {
t.Errorf("FindBestMatchingxdsclient.VirtualHost() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -0,0 +1,164 @@
/*
*
* Copyright 2025 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"fmt"
"net/netip"
"google.golang.org/grpc/internal/envconfig"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
"google.golang.org/protobuf/types/known/anypb"
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
v3gcpauthnpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/gcp_authn/v3"
)
func init() {
if envconfig.XDSHTTPConnectEnabled {
registerMetadataConverter(version.V3AddressURL, proxyAddressConverter{})
}
if envconfig.GCPAuthenticationFilterEnabled {
registerMetadataConverter(version.V3AudienceURL, audienceConverter{})
}
}
var (
// metadataRegistry is a map from proto type to metadataConverter.
metadataRegistry = make(map[string]metadataConverter)
)
// metadataConverter converts xds metadata entries in
// Metadata.typed_filter_metadata into an internal form with the fields relevant
// to gRPC.
type metadataConverter interface {
// convert parses the Any proto into a concrete struct.
convert(*anypb.Any) (any, error)
}
// registerMetadataConverter registers the converter to the map keyed on a proto
// type_url. Must be called at init time. Not thread safe.
func registerMetadataConverter(protoType string, c metadataConverter) {
metadataRegistry[protoType] = c
}
// metadataConverterForType retrieves a converter based on key given.
func metadataConverterForType(typeURL string) metadataConverter {
return metadataRegistry[typeURL]
}
// RegisterMetadataConverterForTesting registers the converter for testing
// purposes and returns a cleanup function to restore the registry to its
// previous state.
func RegisterMetadataConverterForTesting(typeURL string) (func(), error) {
var conv metadataConverter
switch typeURL {
case version.V3AddressURL:
conv = proxyAddressConverter{}
case version.V3AudienceURL:
conv = audienceConverter{}
default:
return nil, fmt.Errorf("unknown typeURL for testing: %s", typeURL)
}
curConverter, found := metadataRegistry[typeURL]
registerMetadataConverter(typeURL, conv)
return func() {
if found {
metadataRegistry[typeURL] = curConverter
return
}
delete(metadataRegistry, typeURL)
}, nil
}
// UnregisterMetadataConverterForTesting unregisters the converter for testing
// purposes and returns a cleanup function to restore the registry to its
// previous state.
func UnregisterMetadataConverterForTesting(typeURL string) func() {
curConverter, found := metadataRegistry[typeURL]
delete(metadataRegistry, typeURL)
return func() {
if found {
metadataRegistry[typeURL] = curConverter
}
}
}
// StructMetadataValue stores the values in a google.protobuf.Struct from
// FilterMetadata.
type StructMetadataValue struct {
// Data stores the parsed JSON representation of a google.protobuf.Struct.
Data map[string]any
}
// ProxyAddressMetadataValue holds the address parsed from the
// envoy.config.core.v3.Address proto message, as specified in gRFC A86.
type ProxyAddressMetadataValue struct {
// Address stores the proxy address configured (A86). It will be in the form
// of host:port. It has to be either IPv6 or IPv4.
Address string
}
// proxyAddressConvertor implements the metadataConverter interface to handle
// the conversion of envoy.config.core.v3.Address protobuf messages into an
// internal representation.
type proxyAddressConverter struct{}
func (proxyAddressConverter) convert(anyProto *anypb.Any) (any, error) {
addressProto := &v3corepb.Address{}
if err := anyProto.UnmarshalTo(addressProto); err != nil {
return nil, fmt.Errorf("failed to unmarshal resource from Any proto: %v", err)
}
socketaddress := addressProto.GetSocketAddress()
if socketaddress == nil {
return nil, fmt.Errorf("no socket_address field in metadata")
}
if _, err := netip.ParseAddr(socketaddress.GetAddress()); err != nil {
return nil, fmt.Errorf("address field is not a valid IPv4 or IPv6 address: %q", socketaddress.GetAddress())
}
portvalue := socketaddress.GetPortValue()
if portvalue == 0 {
return nil, fmt.Errorf("port value not set in socket_address")
}
return ProxyAddressMetadataValue{Address: parseAddress(socketaddress)}, nil
}
// AudienceMetadataValue holds the audience parsed from the
// envoy.extensions.filters.http.gcp_authn.v3.Audience proto message, as
// specified in gRFC A83.
type AudienceMetadataValue struct {
// Audience is the URL of the receiving service that performs token
// authentication.
Audience string
}
// audienceConverter implements the metadataConverter interface to
// handle the conversion of envoy.extensions.filters.http.gcp_authn.v3.Audience
// protobuf messages into an internal representation.
type audienceConverter struct{}
func (audienceConverter) convert(anyProto *anypb.Any) (any, error) {
audienceProto := &v3gcpauthnpb.Audience{}
if err := anyProto.UnmarshalTo(audienceProto); err != nil {
return nil, fmt.Errorf("failed to unmarshal the envoy.extensions.filters.http.gcp_authn.v3.Audience resource from Any proto: %v", err)
}
if audienceProto.GetUrl() == "" {
return nil, fmt.Errorf("empty url field in audience metadata")
}
return AudienceMetadataValue{Audience: audienceProto.GetUrl()}, nil
}

View File

@@ -0,0 +1,259 @@
/*
*
* Copyright 2025 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"testing"
"github.com/google/go-cmp/cmp"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
"google.golang.org/protobuf/types/known/anypb"
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
v3gcpauthnpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/gcp_authn/v3"
)
func (s) TestProxyAddressConverterSuccess(t *testing.T) {
cleanup, err := RegisterMetadataConverterForTesting(version.V3AddressURL)
if err != nil {
t.Fatalf("RegisterMetadataConverterForTesting(%q) failed: %v", version.V3AddressURL, err)
}
t.Cleanup(cleanup)
converter := metadataConverterForType(version.V3AddressURL)
if converter == nil {
t.Fatalf("Converter for %q not found in registry", version.V3AddressURL)
}
tests := []struct {
name string
addr *v3corepb.Address
want ProxyAddressMetadataValue
}{
{
name: "valid_IPv4_address_and_port",
addr: &v3corepb.Address{
Address: &v3corepb.Address_SocketAddress{
SocketAddress: &v3corepb.SocketAddress{
Address: "192.168.1.1",
PortSpecifier: &v3corepb.SocketAddress_PortValue{
PortValue: 8080,
},
},
},
},
want: ProxyAddressMetadataValue{
Address: "192.168.1.1:8080",
},
},
{
name: "valid_full_IPv6_address_and_port",
addr: &v3corepb.Address{
Address: &v3corepb.Address_SocketAddress{
SocketAddress: &v3corepb.SocketAddress{
Address: "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
PortSpecifier: &v3corepb.SocketAddress_PortValue{
PortValue: 9090,
},
},
},
},
want: ProxyAddressMetadataValue{
Address: "[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9090",
},
},
{
name: "valid_shortened_IPv6_address",
addr: &v3corepb.Address{
Address: &v3corepb.Address_SocketAddress{
SocketAddress: &v3corepb.SocketAddress{
Address: "2001:db8::1",
PortSpecifier: &v3corepb.SocketAddress_PortValue{
PortValue: 9090,
},
},
},
},
want: ProxyAddressMetadataValue{
Address: "[2001:db8::1]:9090",
},
},
{
name: "valid_link-local_IPv6_address",
addr: &v3corepb.Address{
Address: &v3corepb.Address_SocketAddress{
SocketAddress: &v3corepb.SocketAddress{
Address: "fe80::1ff:fe23:4567:890a",
PortSpecifier: &v3corepb.SocketAddress_PortValue{
PortValue: 8888,
},
},
},
},
want: ProxyAddressMetadataValue{
Address: "[fe80::1ff:fe23:4567:890a]:8888",
},
},
{
name: "valid_IPv4-mapped_IPv6_address",
addr: &v3corepb.Address{
Address: &v3corepb.Address_SocketAddress{
SocketAddress: &v3corepb.SocketAddress{
Address: "::ffff:192.0.2.128",
PortSpecifier: &v3corepb.SocketAddress_PortValue{
PortValue: 1234,
},
},
},
},
want: ProxyAddressMetadataValue{
Address: "[::ffff:192.0.2.128]:1234",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
anyProto := testutils.MarshalAny(t, tt.addr)
got, err := converter.convert(anyProto)
if err != nil {
t.Fatalf("convert() failed with error: %v", err)
}
if diff := cmp.Diff(tt.want, got, cmp.AllowUnexported(ProxyAddressMetadataValue{})); diff != "" {
t.Errorf("convert() returned unexpected value (-want +got):\n%s", diff)
}
})
}
}
func (s) TestProxyAddressConverterFailure(t *testing.T) {
cleanup, err := RegisterMetadataConverterForTesting(version.V3AddressURL)
if err != nil {
t.Fatalf("RegisterMetadataConverterForTesting(%q) failed: %v", version.V3AddressURL, err)
}
t.Cleanup(cleanup)
converter := metadataConverterForType(version.V3AddressURL)
if converter == nil {
t.Fatalf("Converter for %q not found in registry", version.V3AddressURL)
}
tests := []struct {
name string
addr *v3corepb.Address
wantErr string
}{
{
name: "invalid_address",
addr: &v3corepb.Address{
Address: &v3corepb.Address_SocketAddress{
SocketAddress: &v3corepb.SocketAddress{
Address: "invalid-ip",
},
},
},
wantErr: "address field is not a valid IPv4 or IPv6 address: \"invalid-ip\"",
},
{
name: "missing_socket_address",
addr: &v3corepb.Address{
// No SocketAddress field set.
},
wantErr: "no socket_address field in metadata",
},
{
name: "address_is_not_a_socket_address",
addr: &v3corepb.Address{
Address: &v3corepb.Address_EnvoyInternalAddress{
EnvoyInternalAddress: &v3corepb.EnvoyInternalAddress{
AddressNameSpecifier: &v3corepb.EnvoyInternalAddress_ServerListenerName{
ServerListenerName: "some-internal-listener",
},
},
},
},
wantErr: "no socket_address field in metadata",
},
{
name: "port_value_not_set",
addr: &v3corepb.Address{
Address: &v3corepb.Address_SocketAddress{
SocketAddress: &v3corepb.SocketAddress{
Address: "127.0.0.1",
PortSpecifier: nil,
},
},
},
wantErr: "port value not set in socket_address",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
anyProto := testutils.MarshalAny(t, tt.addr)
_, err := converter.convert(anyProto)
if err == nil || err.Error() != tt.wantErr {
t.Errorf("convert() got error = %v, wantErr = %q", err, tt.wantErr)
}
})
}
}
func (s) TestAudienceConverterSuccess(t *testing.T) {
cleanup, err := RegisterMetadataConverterForTesting(version.V3AudienceURL)
if err != nil {
t.Fatalf("RegisterMetadataConverterForTesting(%q) failed: %v", version.V3AudienceURL, err)
}
t.Cleanup(cleanup)
converter := metadataConverterForType(version.V3AudienceURL)
if converter == nil {
t.Fatalf("Converter for %q not found in registry", version.V3AudienceURL)
}
tests := []struct {
name string
audience *anypb.Any
want AudienceMetadataValue
wantErr string
}{
{
name: "valid_audience",
audience: testutils.MarshalAny(t, &v3gcpauthnpb.Audience{Url: "https://example.com"}),
want: AudienceMetadataValue{Audience: "https://example.com"},
},
{
name: "empty_audience",
audience: testutils.MarshalAny(t, &v3gcpauthnpb.Audience{Url: ""}),
want: AudienceMetadataValue{Audience: ""},
wantErr: "empty url field in audience metadata",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := converter.convert(tt.audience)
if tt.wantErr != "" {
if err == nil || err.Error() != tt.wantErr {
t.Errorf("convert() got error = %v, wantErr = %q", err, tt.wantErr)
}
return
}
if err != nil {
t.Fatalf("convert() failed with error: %v", err)
}
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Errorf("convert(%s) returned unexpected diff (-want +got):\n%s", tt.audience.GetTypeUrl(), diff)
}
})
}
}

View File

@@ -0,0 +1,127 @@
/*
*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"net/url"
"sort"
"strings"
)
// FederationScheme is the scheme of a federation resource name.
const FederationScheme = "xdstp"
// Name contains the parsed component of an xDS resource name.
//
// An xDS resource name is in the format of
// xdstp://[{authority}]/{resource type}/{id/*}?{context parameters}{#processing directive,*}
//
// See
// https://github.com/cncf/xds/blob/main/proposals/TP1-xds-transport-next.md#uri-based-xds-resource-names
// for details, and examples.
type Name struct {
Scheme string
Authority string
Type string
ID string
ContextParams map[string]string
processingDirective string
}
// ParseName splits the name and returns a struct representation of the Name.
//
// If the name isn't a valid new-style xDS name, field ID is set to the input.
// Note that this is not an error, because we still support the old-style
// resource names (those not starting with "xdstp:").
//
// The caller can tell if the parsing is successful by checking the returned
// Scheme.
func ParseName(name string) *Name {
if !strings.Contains(name, "://") {
// Only the long form URL, with ://, is valid.
return &Name{ID: name}
}
parsed, err := url.Parse(name)
if err != nil {
return &Name{ID: name}
}
ret := &Name{
Scheme: parsed.Scheme,
Authority: parsed.Host,
}
split := strings.SplitN(parsed.Path, "/", 3)
if len(split) < 3 {
// Path is in the format of "/type/id". There must be at least 3
// segments after splitting.
return &Name{ID: name}
}
ret.Type = split[1]
ret.ID = split[2]
if len(parsed.Query()) != 0 {
ret.ContextParams = make(map[string]string)
for k, vs := range parsed.Query() {
if len(vs) > 0 {
// We only keep one value of each key. Behavior for multiple values
// is undefined.
ret.ContextParams[k] = vs[0]
}
}
}
// TODO: processing directive (the part comes after "#" in the URL, stored
// in parsed.RawFragment) is kept but not processed. Add support for that
// when it's needed.
ret.processingDirective = parsed.RawFragment
return ret
}
// String returns a canonicalized string of name. The context parameters are
// sorted by the keys.
func (n *Name) String() string {
if n.Scheme == "" {
return n.ID
}
// Sort and build query.
keys := make([]string, 0, len(n.ContextParams))
for k := range n.ContextParams {
keys = append(keys, k)
}
sort.Strings(keys)
var pairs []string
for _, k := range keys {
pairs = append(pairs, strings.Join([]string{k, n.ContextParams[k]}, "="))
}
rawQuery := strings.Join(pairs, "&")
path := n.Type
if n.ID != "" {
path = "/" + path + "/" + n.ID
}
tempURL := &url.URL{
Scheme: n.Scheme,
Host: n.Authority,
Path: path,
RawQuery: rawQuery,
RawFragment: n.processingDirective,
}
return tempURL.String()
}

View File

@@ -0,0 +1,103 @@
/*
*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
)
func TestParseName(t *testing.T) {
tests := []struct {
name string
in string
want *Name
wantStr string
}{
{
name: "old style name",
in: "test-resource",
want: &Name{ID: "test-resource"},
wantStr: "test-resource",
},
{
name: "invalid not url",
in: "a:/b/c",
want: &Name{ID: "a:/b/c"},
wantStr: "a:/b/c",
},
{
name: "invalid no resource type",
in: "xdstp://auth/id",
want: &Name{ID: "xdstp://auth/id"},
wantStr: "xdstp://auth/id",
},
{
name: "valid with no authority",
in: "xdstp:///type/id",
want: &Name{Scheme: "xdstp", Authority: "", Type: "type", ID: "id"},
wantStr: "xdstp:///type/id",
},
{
name: "valid no ctx params",
in: "xdstp://auth/type/id",
want: &Name{Scheme: "xdstp", Authority: "auth", Type: "type", ID: "id"},
wantStr: "xdstp://auth/type/id",
},
{
name: "valid with ctx params",
in: "xdstp://auth/type/id?a=1&b=2",
want: &Name{Scheme: "xdstp", Authority: "auth", Type: "type", ID: "id", ContextParams: map[string]string{"a": "1", "b": "2"}},
wantStr: "xdstp://auth/type/id?a=1&b=2",
},
{
name: "valid with ctx params sorted by keys",
in: "xdstp://auth/type/id?b=2&a=1",
want: &Name{Scheme: "xdstp", Authority: "auth", Type: "type", ID: "id", ContextParams: map[string]string{"a": "1", "b": "2"}},
wantStr: "xdstp://auth/type/id?a=1&b=2",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := ParseName(tt.in)
if !cmp.Equal(got, tt.want, cmpopts.IgnoreFields(Name{}, "processingDirective")) {
t.Errorf("ParseName() = %#v, want %#v", got, tt.want)
}
if gotStr := got.String(); gotStr != tt.wantStr {
t.Errorf("Name.String() = %s, want %s", gotStr, tt.wantStr)
}
})
}
}
// TestNameStringCtxParamsOrder covers the case that if two names differ only in
// context parameter __order__, the parsed name.String() has the same value.
func TestNameStringCtxParamsOrder(t *testing.T) {
const (
a = "xdstp://auth/type/id?a=1&b=2"
b = "xdstp://auth/type/id?b=2&a=1"
)
aParsed := ParseName(a).String()
bParsed := ParseName(b).String()
if aParsed != bParsed {
t.Fatalf("aParsed.String() = %q, bParsed.String() = %q, want them to be the same", aParsed, bParsed)
}
}

View File

@@ -0,0 +1,139 @@
/*
*
* Copyright 2022 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Package xdsresource implements the xDS data model layer.
//
// Provides resource-type specific functionality to unmarshal xDS protos into
// internal data structures that contain only fields gRPC is interested in.
// These internal data structures are passed to components in the xDS stack
// (resolver/balancers/server) that have expressed interest in receiving
// updates to specific resources.
package xdsresource
import (
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/clients/xdsclient"
"google.golang.org/protobuf/types/known/anypb"
)
// Producer contains a single method to discover resource configuration from a
// remote management server using xDS APIs.
//
// The xdsclient package provides a concrete implementation of this interface.
type Producer interface {
// WatchResource uses xDS to discover the resource associated with the
// provided resource name. The resource type implementation determines how
// xDS responses are are deserialized and validated, as received from the
// xDS management server. Upon receipt of a response from the management
// server, an appropriate callback on the watcher is invoked.
WatchResource(typeURL, resourceName string, watcher xdsclient.ResourceWatcher) (cancel func())
}
// ResourceWatcher is notified of the resource updates and errors that are
// received by the xDS client from the management server.
//
// All methods contain a done parameter which should be called when processing
// of the update has completed. For example, if processing a resource requires
// watching new resources, registration of those new watchers should be
// completed before done is called, which can happen after the ResourceWatcher
// method has returned. Failure to call done will prevent the xDS client from
// providing future ResourceWatcher notifications.
type ResourceWatcher interface {
// ResourceChanged indicates a new version of the resource is available.
ResourceChanged(resourceData ResourceData, done func())
// ResourceError indicates an error occurred while trying to fetch or
// decode the associated resource. The previous version of the resource
// should be considered invalid.
ResourceError(err error, done func())
// AmbientError indicates an error occurred after a resource has been
// received that should not modify the use of that resource but may provide
// useful information about the state of the XDSClient for debugging
// purposes. The previous version of the resource should still be
// considered valid.
AmbientError(err error, done func())
}
// TODO: Once the implementation is complete, rename this interface as
// ResourceType and get rid of the existing ResourceType enum.
// Type wraps all resource-type specific functionality. Each supported resource
// type will provide an implementation of this interface.
type Type interface {
// TypeURL is the xDS type URL of this resource type for v3 transport.
TypeURL() string
// TypeName identifies resources in a transport protocol agnostic way. This
// can be used for logging/debugging purposes, as well in cases where the
// resource type name is to be uniquely identified but the actual
// functionality provided by the resource type is not required.
//
// TODO: once Type is renamed to ResourceType, rename TypeName to
// ResourceTypeName.
TypeName() string
// AllResourcesRequiredInSotW indicates whether this resource type requires
// that all resources be present in every SotW response from the server. If
// true, a response that does not include a previously seen resource will be
// interpreted as a deletion of that resource.
AllResourcesRequiredInSotW() bool
// Decode deserializes and validates an xDS resource serialized inside the
// provided `Any` proto, as received from the xDS management server.
//
// If protobuf deserialization fails or resource validation fails,
// returns a non-nil error. Otherwise, returns a fully populated
// DecodeResult.
Decode(*DecodeOptions, *anypb.Any) (*DecodeResult, error)
}
// ResourceData contains the configuration data sent by the xDS management
// server, associated with the resource being watched. Every resource type must
// provide an implementation of this interface to represent the configuration
// received from the xDS management server.
type ResourceData interface {
// RawEqual returns true if the passed in resource data is equal to that of
// the receiver, based on the underlying raw protobuf message.
RawEqual(ResourceData) bool
// ToJSON returns a JSON string representation of the resource data.
ToJSON() string
Raw() *anypb.Any
}
// DecodeOptions wraps the options required by ResourceType implementation for
// decoding configuration received from the xDS management server.
type DecodeOptions struct {
// BootstrapConfig contains the complete bootstrap configuration passed to
// the xDS client. This contains useful data for resource validation.
BootstrapConfig *bootstrap.Config
// ServerConfig contains the server config (from the above bootstrap
// configuration) of the xDS server from which the current resource, for
// which Decode() is being invoked, was received.
ServerConfig *bootstrap.ServerConfig
}
// DecodeResult is the result of a decode operation.
type DecodeResult struct {
// Name is the name of the resource being watched.
Name string
// Resource contains the configuration associated with the resource being
// watched.
Resource ResourceData
}

View File

@@ -0,0 +1,132 @@
/*
*
* Copyright 2022 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"bytes"
"google.golang.org/grpc/internal/xds/bootstrap"
xdsclient "google.golang.org/grpc/internal/xds/clients/xdsclient"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
)
const (
// RouteConfigTypeName represents the transport agnostic name for the
// route config resource.
RouteConfigTypeName = "RouteConfigResource"
)
// routeConfigResourceDecoder is an implementation of the xdsclient.Decoder
// interface for route configuration resources.
type routeConfigResourceDecoder struct {
bootstrapConfig *bootstrap.Config
serverConfigs map[xdsclient.ServerConfig]*bootstrap.ServerConfig
}
func (d *routeConfigResourceDecoder) Decode(resource *xdsclient.AnyProto, opts xdsclient.DecodeOptions) (*xdsclient.DecodeResult, error) {
var serverCfg *bootstrap.ServerConfig
if opts.ServerConfig != nil {
serverCfg = d.serverConfigs[*opts.ServerConfig]
}
name, rc, err := unmarshalRouteConfigResource(resource.ToAny(), d.bootstrapConfig, serverCfg)
if name == "" {
// Name is unset only when protobuf deserialization fails.
return nil, err
}
if err != nil {
// Protobuf deserialization succeeded, but resource validation failed.
return &xdsclient.DecodeResult{
Name: name,
Resource: &RouteConfigResourceData{Resource: RouteConfigUpdate{}},
}, err
}
return &xdsclient.DecodeResult{
Name: name,
Resource: &RouteConfigResourceData{Resource: rc},
}, nil
}
// RouteConfigResourceData is an implementation of the xdsclient.ResourceData
// interface for route configuration resources.
type RouteConfigResourceData struct {
Resource RouteConfigUpdate
}
// Equal returns true if other is equal to er.
func (r *RouteConfigResourceData) Equal(other xdsclient.ResourceData) bool {
if other == nil {
return false
}
return bytes.Equal(r.Bytes(), other.Bytes())
}
// Bytes returns the protobuf serialized bytes of the route config resource proto.
func (r *RouteConfigResourceData) Bytes() []byte {
return r.Resource.Raw.GetValue()
}
// RouteConfigWatcher wraps the callbacks to be invoked for different
// events corresponding to the route configuration resource being watched. gRFC
// A88 contains an exhaustive list of what method is invoked under what
// conditions.
type RouteConfigWatcher interface {
// ResourceChanged indicates a new version of the resource is available.
ResourceChanged(resource *RouteConfigUpdate, done func())
// ResourceError indicates an error occurred while trying to fetch or
// decode the associated resource. The previous version of the resource
// should be considered invalid.
ResourceError(err error, done func())
// AmbientError indicates an error occurred after a resource has been
// received that should not modify the use of that resource but may provide
// useful information about the state of the XDSClient for debugging
// purposes. The previous version of the resource should still be
// considered valid.
AmbientError(err error, done func())
}
type delegatingRouteConfigWatcher struct {
watcher RouteConfigWatcher
}
func (d *delegatingRouteConfigWatcher) ResourceChanged(data xdsclient.ResourceData, onDone func()) {
rc := data.(*RouteConfigResourceData)
d.watcher.ResourceChanged(&rc.Resource, onDone)
}
func (d *delegatingRouteConfigWatcher) ResourceError(err error, onDone func()) {
d.watcher.ResourceError(err, onDone)
}
func (d *delegatingRouteConfigWatcher) AmbientError(err error, onDone func()) {
d.watcher.AmbientError(err, onDone)
}
// WatchRouteConfig uses xDS to discover the configuration associated with the
// provided route configuration resource name.
func WatchRouteConfig(p Producer, name string, w RouteConfigWatcher) (cancel func()) {
return p.WatchResource(version.V3RouteConfigURL, name, &delegatingRouteConfigWatcher{watcher: w})
}
// NewRouteConfigResourceTypeDecoder returns a xdsclient.Decoder that wraps
// the xdsresource.routeConfigType.
func NewRouteConfigResourceTypeDecoder(bc *bootstrap.Config, gServerCfgMap map[xdsclient.ServerConfig]*bootstrap.ServerConfig) xdsclient.Decoder {
return &routeConfigResourceDecoder{bootstrapConfig: bc, serverConfigs: gServerCfgMap}
}

View File

@@ -0,0 +1,124 @@
/*
*
* Copyright 2020 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"net/netip"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"google.golang.org/grpc/internal/grpctest"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/testutils/xds/e2e"
"google.golang.org/grpc/internal/xds/httpfilter"
"google.golang.org/grpc/internal/xds/httpfilter/router"
"google.golang.org/protobuf/testing/protocmp"
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
)
type s struct {
grpctest.Tester
}
func Test(t *testing.T) {
grpctest.RunSubTests(t, s{})
}
const defaultTestTimeout = 10 * time.Second
var (
cmpOpts = cmp.Options{
cmpopts.EquateEmpty(),
cmpopts.EquateComparable(netip.Addr{}, netip.Prefix{}),
cmp.FilterValues(func(_, _ error) bool { return true }, cmpopts.EquateErrors()),
cmp.Comparer(func(_, _ time.Time) bool { return true }),
protocmp.Transform(),
}
routeConfig = &v3routepb.RouteConfiguration{
Name: "routeName",
VirtualHosts: []*v3routepb.VirtualHost{{
Domains: []string{"lds.target.good:3333"},
Routes: []*v3routepb.Route{{
Match: &v3routepb.RouteMatch{
PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"},
},
Action: &v3routepb.Route_NonForwardingAction{},
}}}}}
inlineRouteConfig = &RouteConfigUpdate{
VirtualHosts: []*VirtualHost{{
Domains: []string{"lds.target.good:3333"},
Routes: []*Route{{Prefix: newStringP("/"), ActionType: RouteActionNonForwardingAction}},
}}}
validServerSideHTTPFilter1 = &v3httppb.HttpFilter{
Name: "serverOnlyCustomFilter",
ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: serverOnlyCustomFilterConfig},
}
validServerSideHTTPFilter2 = &v3httppb.HttpFilter{
Name: "serverOnlyCustomFilter2",
ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: serverOnlyCustomFilterConfig},
}
emptyRouterFilter = e2e.RouterHTTPFilter
localSocketAddress = &v3corepb.Address{
Address: &v3corepb.Address_SocketAddress{
SocketAddress: &v3corepb.SocketAddress{
Address: "0.0.0.0",
PortSpecifier: &v3corepb.SocketAddress_PortValue{
PortValue: 9999,
},
},
},
}
)
func makeRouterFilter(t *testing.T) HTTPFilter {
t.Helper()
routerBuilder := httpfilter.Get(router.TypeURL)
routerConfig, err := routerBuilder.ParseFilterConfig(testutils.MarshalAny(t, &v3routerpb.Router{}), httpfilter.ParseOptions{})
if err != nil {
t.Fatalf("Failed to parse Router filter configuration: %v", err)
}
return HTTPFilter{Name: "router", Filter: routerBuilder, Config: routerConfig}
}
func makeRouterFilterList(t *testing.T) []HTTPFilter {
return []HTTPFilter{makeRouterFilter(t)}
}
func emptyValidNetworkFilters(t *testing.T) []*v3listenerpb.Filter {
return []*v3listenerpb.Filter{
{
Name: "filter-1",
ConfigType: &v3listenerpb.Filter_TypedConfig{
TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
RouteConfig: routeConfig,
},
HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter},
}),
},
},
}
}

View File

@@ -0,0 +1,575 @@
/*
*
* Copyright 2023 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
// Package tests_test contains test cases for unmarshalling of CDS resources.
package tests_test
import (
"encoding/json"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"google.golang.org/grpc/balancer/leastrequest"
"google.golang.org/grpc/internal/balancer/stub"
"google.golang.org/grpc/internal/grpctest"
iringhash "google.golang.org/grpc/internal/ringhash"
iserviceconfig "google.golang.org/grpc/internal/serviceconfig"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/testutils/xds/e2e"
xdsinternal "google.golang.org/grpc/internal/xds"
"google.golang.org/grpc/internal/xds/balancer/wrrlocality"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
"google.golang.org/grpc/serviceconfig"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"google.golang.org/protobuf/types/known/structpb"
"google.golang.org/protobuf/types/known/wrapperspb"
v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3"
v3ringhashpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/ring_hash/v3"
v3roundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/round_robin/v3"
v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3"
_ "google.golang.org/grpc/balancer/roundrobin" // To register round_robin load balancer.
_ "google.golang.org/grpc/xds" // Register the xDS LB Registry Converters.
)
type s struct {
grpctest.Tester
}
func Test(t *testing.T) {
grpctest.RunSubTests(t, s{})
}
const (
clusterName = "clusterName"
serviceName = "service"
)
func wrrLocality(t *testing.T, m proto.Message) *v3wrrlocalitypb.WrrLocality {
return &v3wrrlocalitypb.WrrLocality{
EndpointPickingPolicy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, m),
},
},
},
},
}
}
func wrrLocalityAny(t *testing.T, m proto.Message) *anypb.Any {
return testutils.MarshalAny(t, wrrLocality(t, m))
}
type customLBConfig struct {
serviceconfig.LoadBalancingConfig
}
// We have this test in a separate test package in order to not take a
// dependency on the internal xDS balancer packages within the xDS Client.
func (s) TestValidateCluster_Success(t *testing.T) {
const customLBPolicyName = "myorg.MyCustomLeastRequestPolicy"
stub.Register(customLBPolicyName, stub.BalancerFuncs{
ParseConfig: func(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
return customLBConfig{}, nil
},
})
serverCfg, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: "test-server"})
if err != nil {
t.Fatalf("Failed to create server config for testing: %v", err)
}
tests := []struct {
name string
cluster *v3clusterpb.Cluster
serverCfg *bootstrap.ServerConfig
wantUpdate xdsresource.ClusterUpdate
wantLBConfig *iserviceconfig.BalancerConfig
}{
{
name: "happy-case-logical-dns",
cluster: &v3clusterpb.Cluster{
Name: clusterName,
ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_LOGICAL_DNS},
LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
LoadAssignment: &v3endpointpb.ClusterLoadAssignment{
Endpoints: []*v3endpointpb.LocalityLbEndpoints{{
LbEndpoints: []*v3endpointpb.LbEndpoint{{
HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{
Endpoint: &v3endpointpb.Endpoint{
Address: &v3corepb.Address{
Address: &v3corepb.Address_SocketAddress{
SocketAddress: &v3corepb.SocketAddress{
Address: "dns_host",
PortSpecifier: &v3corepb.SocketAddress_PortValue{
PortValue: 8080,
},
},
},
},
},
},
}},
}},
},
},
wantUpdate: xdsresource.ClusterUpdate{
ClusterName: clusterName,
ClusterType: xdsresource.ClusterTypeLogicalDNS,
DNSHostName: "dns_host:8080",
TelemetryLabels: xdsinternal.UnknownCSMLabels,
},
wantLBConfig: &iserviceconfig.BalancerConfig{
Name: wrrlocality.Name,
Config: &wrrlocality.LBConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: "round_robin",
},
},
},
},
{
name: "happy-case-aggregate-v3",
cluster: &v3clusterpb.Cluster{
Name: clusterName,
ClusterDiscoveryType: &v3clusterpb.Cluster_ClusterType{
ClusterType: &v3clusterpb.Cluster_CustomClusterType{
Name: "envoy.clusters.aggregate",
TypedConfig: testutils.MarshalAny(t, &v3aggregateclusterpb.ClusterConfig{
Clusters: []string{"a", "b", "c"},
}),
},
},
LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
},
wantUpdate: xdsresource.ClusterUpdate{
ClusterName: clusterName,
ClusterType: xdsresource.ClusterTypeAggregate,
PrioritizedClusterNames: []string{"a", "b", "c"},
TelemetryLabels: xdsinternal.UnknownCSMLabels,
},
wantLBConfig: &iserviceconfig.BalancerConfig{
Name: wrrlocality.Name,
Config: &wrrlocality.LBConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: "round_robin",
},
},
},
},
{
name: "happy-case-no-service-name-no-lrs",
cluster: e2e.DefaultCluster(clusterName, "", e2e.SecurityLevelNone),
wantUpdate: xdsresource.ClusterUpdate{
ClusterName: clusterName,
TelemetryLabels: xdsinternal.UnknownCSMLabels,
},
wantLBConfig: &iserviceconfig.BalancerConfig{
Name: wrrlocality.Name,
Config: &wrrlocality.LBConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: "round_robin",
},
},
},
},
{
name: "happy-case-no-lrs",
cluster: e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone),
wantUpdate: xdsresource.ClusterUpdate{
ClusterName: clusterName,
EDSServiceName: serviceName,
TelemetryLabels: xdsinternal.UnknownCSMLabels,
},
wantLBConfig: &iserviceconfig.BalancerConfig{
Name: wrrlocality.Name,
Config: &wrrlocality.LBConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: "round_robin",
},
},
},
},
{
name: "happiest-case-with-lrs",
cluster: e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
ClusterName: clusterName,
ServiceName: serviceName,
EnableLRS: true,
}),
serverCfg: serverCfg,
wantUpdate: xdsresource.ClusterUpdate{
ClusterName: clusterName,
EDSServiceName: serviceName,
LRSServerConfig: serverCfg,
TelemetryLabels: xdsinternal.UnknownCSMLabels,
},
wantLBConfig: &iserviceconfig.BalancerConfig{
Name: wrrlocality.Name,
Config: &wrrlocality.LBConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: "round_robin",
},
},
},
},
{
name: "happiest-case-with-circuitbreakers",
cluster: func() *v3clusterpb.Cluster {
c := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
ClusterName: clusterName,
ServiceName: serviceName,
EnableLRS: true,
})
c.CircuitBreakers = &v3clusterpb.CircuitBreakers{
Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{
{
Priority: v3corepb.RoutingPriority_DEFAULT,
MaxRequests: wrapperspb.UInt32(512),
},
{
Priority: v3corepb.RoutingPriority_HIGH,
MaxRequests: nil,
},
},
}
return c
}(),
serverCfg: serverCfg,
wantUpdate: xdsresource.ClusterUpdate{
ClusterName: clusterName,
EDSServiceName: serviceName,
LRSServerConfig: serverCfg,
MaxRequests: func() *uint32 { i := uint32(512); return &i }(),
TelemetryLabels: xdsinternal.UnknownCSMLabels,
},
wantLBConfig: &iserviceconfig.BalancerConfig{
Name: wrrlocality.Name,
Config: &wrrlocality.LBConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: "round_robin",
},
},
},
},
{
name: "happiest-case-with-ring-hash-lb-policy-with-default-config",
cluster: func() *v3clusterpb.Cluster {
c := e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)
c.LbPolicy = v3clusterpb.Cluster_RING_HASH
return c
}(),
wantUpdate: xdsresource.ClusterUpdate{
ClusterName: clusterName,
EDSServiceName: serviceName,
TelemetryLabels: xdsinternal.UnknownCSMLabels,
},
wantLBConfig: &iserviceconfig.BalancerConfig{
Name: "ring_hash_experimental",
Config: &iringhash.LBConfig{
MinRingSize: 1024,
MaxRingSize: 4096,
},
},
},
{
name: "happiest-case-with-least-request-lb-policy-with-default-config",
cluster: &v3clusterpb.Cluster{
Name: clusterName,
ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
EdsConfig: &v3corepb.ConfigSource{
ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
Ads: &v3corepb.AggregatedConfigSource{},
},
},
ServiceName: serviceName,
},
LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST,
},
wantUpdate: xdsresource.ClusterUpdate{
ClusterName: clusterName,
EDSServiceName: serviceName,
TelemetryLabels: xdsinternal.UnknownCSMLabels,
},
wantLBConfig: &iserviceconfig.BalancerConfig{
Name: "least_request_experimental",
Config: &leastrequest.LBConfig{
ChoiceCount: 2,
},
},
},
{
name: "happiest-case-with-ring-hash-lb-policy-with-none-default-config",
cluster: func() *v3clusterpb.Cluster {
c := e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)
c.LbPolicy = v3clusterpb.Cluster_RING_HASH
c.LbConfig = &v3clusterpb.Cluster_RingHashLbConfig_{
RingHashLbConfig: &v3clusterpb.Cluster_RingHashLbConfig{
MinimumRingSize: wrapperspb.UInt64(10),
MaximumRingSize: wrapperspb.UInt64(100),
},
}
return c
}(),
wantUpdate: xdsresource.ClusterUpdate{
ClusterName: clusterName,
EDSServiceName: serviceName,
TelemetryLabels: xdsinternal.UnknownCSMLabels,
},
wantLBConfig: &iserviceconfig.BalancerConfig{
Name: "ring_hash_experimental",
Config: &iringhash.LBConfig{
MinRingSize: 10,
MaxRingSize: 100,
},
},
},
{
name: "happiest-case-with-least-request-lb-policy-with-none-default-config",
cluster: &v3clusterpb.Cluster{
Name: clusterName,
ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
EdsConfig: &v3corepb.ConfigSource{
ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
Ads: &v3corepb.AggregatedConfigSource{},
},
},
ServiceName: serviceName,
},
LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST,
LbConfig: &v3clusterpb.Cluster_LeastRequestLbConfig_{
LeastRequestLbConfig: &v3clusterpb.Cluster_LeastRequestLbConfig{
ChoiceCount: wrapperspb.UInt32(3),
},
},
},
wantUpdate: xdsresource.ClusterUpdate{
ClusterName: clusterName,
EDSServiceName: serviceName,
TelemetryLabels: xdsinternal.UnknownCSMLabels,
},
wantLBConfig: &iserviceconfig.BalancerConfig{
Name: "least_request_experimental",
Config: &leastrequest.LBConfig{
ChoiceCount: 3,
},
},
},
{
name: "happiest-case-with-ring-hash-lb-policy-configured-through-LoadBalancingPolicy",
cluster: &v3clusterpb.Cluster{
Name: clusterName,
ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
EdsConfig: &v3corepb.ConfigSource{
ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
Ads: &v3corepb.AggregatedConfigSource{},
},
},
ServiceName: serviceName,
},
LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v3ringhashpb.RingHash{
HashFunction: v3ringhashpb.RingHash_XX_HASH,
MinimumRingSize: wrapperspb.UInt64(10),
MaximumRingSize: wrapperspb.UInt64(100),
}),
},
},
},
},
},
wantUpdate: xdsresource.ClusterUpdate{
ClusterName: clusterName,
EDSServiceName: serviceName,
TelemetryLabels: xdsinternal.UnknownCSMLabels,
},
wantLBConfig: &iserviceconfig.BalancerConfig{
Name: "ring_hash_experimental",
Config: &iringhash.LBConfig{
MinRingSize: 10,
MaxRingSize: 100,
},
},
},
{
name: "happiest-case-with-wrrlocality-rr-child-configured-through-LoadBalancingPolicy",
cluster: &v3clusterpb.Cluster{
Name: clusterName,
ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
EdsConfig: &v3corepb.ConfigSource{
ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
Ads: &v3corepb.AggregatedConfigSource{},
},
},
ServiceName: serviceName,
},
LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: wrrLocalityAny(t, &v3roundrobinpb.RoundRobin{}),
},
},
},
},
},
wantUpdate: xdsresource.ClusterUpdate{
ClusterName: clusterName,
EDSServiceName: serviceName,
TelemetryLabels: xdsinternal.UnknownCSMLabels,
},
wantLBConfig: &iserviceconfig.BalancerConfig{
Name: wrrlocality.Name,
Config: &wrrlocality.LBConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: "round_robin",
},
},
},
},
{
name: "happiest-case-with-custom-lb-configured-through-LoadBalancingPolicy",
cluster: &v3clusterpb.Cluster{
Name: clusterName,
ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
EdsConfig: &v3corepb.ConfigSource{
ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
Ads: &v3corepb.AggregatedConfigSource{},
},
},
ServiceName: serviceName,
},
LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: wrrLocalityAny(t, &v3xdsxdstypepb.TypedStruct{
TypeUrl: "type.googleapis.com/myorg.MyCustomLeastRequestPolicy",
Value: &structpb.Struct{},
}),
},
},
},
},
},
wantUpdate: xdsresource.ClusterUpdate{
ClusterName: clusterName,
EDSServiceName: serviceName,
TelemetryLabels: xdsinternal.UnknownCSMLabels,
},
wantLBConfig: &iserviceconfig.BalancerConfig{
Name: wrrlocality.Name,
Config: &wrrlocality.LBConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: "myorg.MyCustomLeastRequestPolicy",
Config: customLBConfig{},
},
},
},
},
{
name: "load-balancing-policy-takes-precedence-over-lb-policy-and-enum",
cluster: &v3clusterpb.Cluster{
Name: clusterName,
ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
EdsConfig: &v3corepb.ConfigSource{
ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
Ads: &v3corepb.AggregatedConfigSource{},
},
},
ServiceName: serviceName,
},
LbPolicy: v3clusterpb.Cluster_RING_HASH,
LbConfig: &v3clusterpb.Cluster_RingHashLbConfig_{
RingHashLbConfig: &v3clusterpb.Cluster_RingHashLbConfig{
MinimumRingSize: wrapperspb.UInt64(20),
MaximumRingSize: wrapperspb.UInt64(200),
},
},
LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{
Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
{
TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
TypedConfig: testutils.MarshalAny(t, &v3ringhashpb.RingHash{
HashFunction: v3ringhashpb.RingHash_XX_HASH,
MinimumRingSize: wrapperspb.UInt64(10),
MaximumRingSize: wrapperspb.UInt64(100),
}),
},
},
},
},
},
wantUpdate: xdsresource.ClusterUpdate{
ClusterName: clusterName,
EDSServiceName: serviceName,
TelemetryLabels: xdsinternal.UnknownCSMLabels,
},
wantLBConfig: &iserviceconfig.BalancerConfig{
Name: "ring_hash_experimental",
Config: &iringhash.LBConfig{
MinRingSize: 10,
MaxRingSize: 100,
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
update, err := xdsresource.ValidateClusterAndConstructClusterUpdateForTesting(test.cluster, test.serverCfg)
if err != nil {
t.Errorf("validateClusterAndConstructClusterUpdate(%+v) failed: %v", test.cluster, err)
}
// Ignore the raw JSON string into the cluster update. JSON bytes
// are nondeterministic (whitespace etc.) so we cannot reliably
// compare JSON bytes in a test. Thus, marshal into a Balancer
// Config struct and compare on that. Only need to test this JSON
// emission here, as this covers the possible output space.
if diff := cmp.Diff(update, test.wantUpdate, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(xdsresource.ClusterUpdate{}, "LBPolicy")); diff != "" {
t.Errorf("validateClusterAndConstructClusterUpdate(%+v) got diff: %v (-got, +want)", test.cluster, diff)
}
bc := &iserviceconfig.BalancerConfig{}
if err := json.Unmarshal(update.LBPolicy, bc); err != nil {
t.Fatalf("failed to unmarshal JSON: %v", err)
}
if diff := cmp.Diff(bc, test.wantLBConfig); diff != "" {
t.Fatalf("update.LBConfig got unexpected output, diff (-got +want): %v", diff)
}
})
}
}

View File

@@ -0,0 +1,135 @@
/*
*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"time"
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
)
// UpdateValidatorFunc performs validations on update structs using
// context/logic available at the xdsClient layer. Since these validation are
// performed on internal update structs, they can be shared between different
// API clients.
type UpdateValidatorFunc func(any) error
// UpdateMetadata contains the metadata for each update, including timestamp,
// raw message, and so on.
type UpdateMetadata struct {
// Status is the status of this resource, e.g. ACKed, NACKed, or
// Not_exist(removed).
Status ServiceStatus
// Version is the version of the xds response. Note that this is the version
// of the resource in use (previous ACKed). If a response is NACKed, the
// NACKed version is in ErrState.
Version string
// Timestamp is when the response is received.
Timestamp time.Time
// ErrState is set when the update is NACKed.
ErrState *UpdateErrorMetadata
}
// IsListenerResource returns true if the provider URL corresponds to an xDS
// Listener resource.
func IsListenerResource(url string) bool {
return url == version.V3ListenerURL
}
// IsHTTPConnManagerResource returns true if the provider URL corresponds to an xDS
// HTTPConnManager resource.
func IsHTTPConnManagerResource(url string) bool {
return url == version.V3HTTPConnManagerURL
}
// IsRouteConfigResource returns true if the provider URL corresponds to an xDS
// RouteConfig resource.
func IsRouteConfigResource(url string) bool {
return url == version.V3RouteConfigURL
}
// IsClusterResource returns true if the provider URL corresponds to an xDS
// Cluster resource.
func IsClusterResource(url string) bool {
return url == version.V3ClusterURL
}
// IsEndpointsResource returns true if the provider URL corresponds to an xDS
// Endpoints resource.
func IsEndpointsResource(url string) bool {
return url == version.V3EndpointsURL
}
// UnwrapResource unwraps and returns the inner resource if it's in a resource
// wrapper. The original resource is returned if it's not wrapped.
func UnwrapResource(r *anypb.Any) (*anypb.Any, error) {
url := r.GetTypeUrl()
if url != version.V3ResourceWrapperURL {
// Not wrapped.
return r, nil
}
inner := &v3discoverypb.Resource{}
if err := proto.Unmarshal(r.GetValue(), inner); err != nil {
return nil, err
}
return inner.Resource, nil
}
// ServiceStatus is the status of the update.
type ServiceStatus int
const (
// ServiceStatusUnknown is the default state, before a watch is started for
// the resource.
ServiceStatusUnknown ServiceStatus = iota
// ServiceStatusRequested is when the watch is started, but before and
// response is received.
ServiceStatusRequested
// ServiceStatusNotExist is when the resource doesn't exist in
// state-of-the-world responses (e.g. LDS and CDS), which means the resource
// is removed by the management server.
ServiceStatusNotExist // Resource is removed in the server, in LDS/CDS.
// ServiceStatusACKed is when the resource is ACKed.
ServiceStatusACKed
// ServiceStatusNACKed is when the resource is NACKed.
ServiceStatusNACKed
)
// UpdateErrorMetadata is part of UpdateMetadata. It contains the error state
// when a response is NACKed.
type UpdateErrorMetadata struct {
// Version is the version of the NACKed response.
Version string
// Err contains why the response was NACKed.
Err error
// Timestamp is when the NACKed response was received.
Timestamp time.Time
}
// UpdateWithMD contains the raw message of the update and the metadata,
// including version, raw message, timestamp.
//
// This is to be used for config dump and CSDS, not directly by users (like
// resolvers/balancers).
type UpdateWithMD struct {
MD UpdateMetadata
Raw *anypb.Any
}

View File

@@ -0,0 +1,228 @@
/*
*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"encoding/json"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/matcher"
"google.golang.org/protobuf/types/known/anypb"
)
// ClusterType is the type of cluster from a received CDS response.
type ClusterType int
const (
// ClusterTypeEDS represents the EDS cluster type, which will delegate endpoint
// discovery to the management server.
ClusterTypeEDS ClusterType = iota
// ClusterTypeLogicalDNS represents the Logical DNS cluster type, which essentially
// maps to the gRPC behavior of using the DNS resolver with pick_first LB policy.
ClusterTypeLogicalDNS
// ClusterTypeAggregate represents the Aggregate Cluster type, which provides a
// prioritized list of clusters to use. It is used for failover between clusters
// with a different configuration.
ClusterTypeAggregate
)
// ClusterUpdate contains information from a received CDS response, which is of
// interest to the registered CDS watcher.
type ClusterUpdate struct {
ClusterType ClusterType
// ClusterName is the clusterName being watched for through CDS.
ClusterName string
// EDSServiceName is an optional name for EDS. If it's not set, the balancer
// should watch ClusterName for the EDS resources.
EDSServiceName string
// LRSServerConfig contains configuration about the xDS server that sent
// this cluster resource. This is also the server where load reports are to
// be sent, for this cluster.
LRSServerConfig *bootstrap.ServerConfig
// SecurityCfg contains security configuration sent by the control plane.
SecurityCfg *SecurityConfig
// MaxRequests for circuit breaking, if any (otherwise nil).
MaxRequests *uint32
// DNSHostName is used only for cluster type DNS. It's the DNS name to
// resolve in "host:port" form
DNSHostName string
// PrioritizedClusterNames is used only for cluster type aggregate. It represents
// a prioritized list of cluster names.
PrioritizedClusterNames []string
// LBPolicy represents the locality and endpoint picking policy in JSON,
// which will be the child policy of xds_cluster_impl.
LBPolicy json.RawMessage
// OutlierDetection is the outlier detection configuration for this cluster.
// If nil, it means this cluster does not use the outlier detection feature.
OutlierDetection json.RawMessage
// Raw is the resource from the xds response.
Raw *anypb.Any
// TelemetryLabels are the string valued metadata of filter_metadata type
// "com.google.csm.telemetry_labels" with keys "service_name" or
// "service_namespace".
TelemetryLabels map[string]string
// LRSReportEndpointMetrics specifies the subset of ORCA metrics that
// should be propagated to the LRS server.
LRSReportEndpointMetrics *LRSReportEndpointMetricsConfig
// Metadata contains the metadata from the cluster resource.
Metadata map[string]any
// IsHTTP11ProxyEnabled is true if the cluster transport socket is wrapped
// in Http11ProxyUpstreamTransport.
IsHTTP11ProxyEnabled bool
}
// LRSReportEndpointMetricsConfig holds the configuration for propagating ORCA
// metrics to the LRS server.
type LRSReportEndpointMetricsConfig struct {
CPUUtilization bool
MemUtilization bool
ApplicationUtilization bool
// NamedMetricsAll specifies whether all named metrics should be propagated.
// If true, NamedMetrics is ignored.
NamedMetricsAll bool
// NamedMetrics specifies the set of named metrics to propagate when
// NamedMetricsAll is false.
NamedMetrics map[string]struct{}
}
// Equal returns whether the two LRSReportEndpointMetricsConfig configurations
// are identical.
func (bmp *LRSReportEndpointMetricsConfig) Equal(other *LRSReportEndpointMetricsConfig) bool {
switch {
case bmp == nil && other == nil:
return true
case (bmp != nil) != (other != nil):
return false
}
if bmp.CPUUtilization != other.CPUUtilization ||
bmp.MemUtilization != other.MemUtilization ||
bmp.ApplicationUtilization != other.ApplicationUtilization ||
bmp.NamedMetricsAll != other.NamedMetricsAll {
return false
}
if len(bmp.NamedMetrics) != len(other.NamedMetrics) {
return false
}
for k := range bmp.NamedMetrics {
if _, ok := other.NamedMetrics[k]; !ok {
return false
}
}
return true
}
// SecurityConfig contains the security configuration received as part of the
// Cluster resource on the client-side, and as part of the Listener resource on
// the server-side.
type SecurityConfig struct {
// RootInstanceName identifies the certProvider plugin to be used to fetch
// root certificates. This instance name will be resolved to the plugin name
// and its associated configuration from the certificate_providers field of
// the bootstrap file.
RootInstanceName string
// RootCertName is the certificate name to be passed to the plugin (looked
// up from the bootstrap file) while fetching root certificates.
RootCertName string
// IdentityInstanceName identifies the certProvider plugin to be used to
// fetch identity certificates. This instance name will be resolved to the
// plugin name and its associated configuration from the
// certificate_providers field of the bootstrap file.
IdentityInstanceName string
// IdentityCertName is the certificate name to be passed to the plugin
// (looked up from the bootstrap file) while fetching identity certificates.
IdentityCertName string
// SubjectAltNameMatchers is an optional list of match criteria for SANs
// specified on the peer certificate. Used only on the client-side.
//
// Some intricacies:
// - If this field is empty, then any peer certificate is accepted.
// - If the peer certificate contains a wildcard DNS SAN, and an `exact`
// matcher is configured, a wildcard DNS match is performed instead of a
// regular string comparison.
SubjectAltNameMatchers []matcher.StringMatcher
// RequireClientCert indicates if the server handshake process expects the
// client to present a certificate. Set to true when performing mTLS. Used
// only on the server-side.
RequireClientCert bool
// UseSystemRootCerts indicates that the client should use system root
// certificates to validate the server certificate. This field is mutually
// exclusive with RootCertName and RootInstanceName. Validation performed
// after unmarshalling xDS resources ensures that this field is set only
// when both RootCertName and RootInstanceName are empty.
UseSystemRootCerts bool
// SNI is the string to be used as the Server Name when creating TLS
// configurations for the handshake. An empty string for SNI value will be
// treated as SNI not specified.
SNI string
// UseAutoHostSNI indicates whether to set the ServerName for the TLS handshake
// configuration to the hostname (if available). The host is the DNS
// hostname for DNS clusters, or Endpoint.hostname for EDS clusters. The
// port will not be included in the SNI value.
UseAutoHostSNI bool
// AutoSNISANValidation indicates whether to replace any Subject Alternative
// Name (SAN) matchers with a validation for a DNS SAN matching the SNI
// value sent. This validation uses the SNI being set in the TLS
// configuration, regardless of how the SNI is determined.
AutoSNISANValidation bool
}
// Equal returns true if sc is equal to other.
func (sc *SecurityConfig) Equal(other *SecurityConfig) bool {
switch {
case sc == nil && other == nil:
return true
case (sc != nil) != (other != nil):
return false
}
switch {
case sc.RootInstanceName != other.RootInstanceName:
return false
case sc.RootCertName != other.RootCertName:
return false
case sc.IdentityInstanceName != other.IdentityInstanceName:
return false
case sc.IdentityCertName != other.IdentityCertName:
return false
case sc.RequireClientCert != other.RequireClientCert:
return false
case sc.UseSystemRootCerts != other.UseSystemRootCerts:
return false
case sc.SNI != other.SNI:
return false
case sc.UseAutoHostSNI != other.UseAutoHostSNI:
return false
case sc.AutoSNISANValidation != other.AutoSNISANValidation:
return false
default:
if len(sc.SubjectAltNameMatchers) != len(other.SubjectAltNameMatchers) {
return false
}
for i := 0; i < len(sc.SubjectAltNameMatchers); i++ {
if !sc.SubjectAltNameMatchers[i].Equal(other.SubjectAltNameMatchers[i]) {
return false
}
}
}
return true
}

View File

@@ -0,0 +1,78 @@
/*
*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"google.golang.org/grpc/internal/xds/clients"
"google.golang.org/grpc/resolver"
"google.golang.org/protobuf/types/known/anypb"
)
// OverloadDropConfig contains the config to drop overloads.
type OverloadDropConfig struct {
Category string
Numerator uint32
Denominator uint32
}
// EndpointHealthStatus represents the health status of an endpoint.
type EndpointHealthStatus int32
const (
// EndpointHealthStatusUnknown represents HealthStatus UNKNOWN.
EndpointHealthStatusUnknown EndpointHealthStatus = iota
// EndpointHealthStatusHealthy represents HealthStatus HEALTHY.
EndpointHealthStatusHealthy
// EndpointHealthStatusUnhealthy represents HealthStatus UNHEALTHY.
EndpointHealthStatusUnhealthy
// EndpointHealthStatusDraining represents HealthStatus DRAINING.
EndpointHealthStatusDraining
// EndpointHealthStatusTimeout represents HealthStatus TIMEOUT.
EndpointHealthStatusTimeout
// EndpointHealthStatusDegraded represents HealthStatus DEGRADED.
EndpointHealthStatusDegraded
)
// Endpoint contains information of an endpoint.
type Endpoint struct {
ResolverEndpoint resolver.Endpoint
HealthStatus EndpointHealthStatus
Weight uint32
Metadata map[string]any
}
// Locality contains information of a locality.
type Locality struct {
Endpoints []Endpoint
ID clients.Locality
Priority uint32
Weight uint32
Metadata map[string]any
}
// EndpointsUpdate contains an EDS update.
type EndpointsUpdate struct {
Drops []OverloadDropConfig
// Localities in the EDS response with `load_balancing_weight` field not set
// or explicitly set to 0 are ignored while parsing the resource, and
// therefore do not show up here.
Localities []Locality
// Raw is the resource from the xds response.
Raw *anypb.Any
}

View File

@@ -0,0 +1,92 @@
/*
*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"time"
"google.golang.org/grpc/internal/xds/httpfilter"
"google.golang.org/protobuf/types/known/anypb"
)
// ListenerUpdate contains information received in an LDS response, which is of
// interest to the registered LDS watcher.
type ListenerUpdate struct {
// APIListener contains the HTTP connection manager configuration.
APIListener *HTTPConnectionManagerConfig
// TCPListener contains inbound listener configuration.
TCPListener *InboundListenerConfig
// Raw is the resource from the xds response.
Raw *anypb.Any
}
// HTTPConnectionManagerConfig contains the HTTP connection manager configuration.
type HTTPConnectionManagerConfig struct {
// RouteConfigName is the route configuration name corresponding to the
// target which is being watched through LDS.
//
// Exactly one of RouteConfigName and InlineRouteConfig is set.
RouteConfigName string
// InlineRouteConfig is the inline route configuration (RDS response)
// returned inside LDS.
//
// Exactly one of RouteConfigName and InlineRouteConfig is set.
InlineRouteConfig *RouteConfigUpdate
// MaxStreamDuration contains the HTTP connection manager's
// common_http_protocol_options.max_stream_duration field, or zero if
// unset.
MaxStreamDuration time.Duration
// HTTPFilters is a list of HTTP filters (name, config) from the LDS
// response.
HTTPFilters []HTTPFilter
}
// HTTPFilter represents one HTTP filter from an LDS response's HTTP connection
// manager field.
type HTTPFilter struct {
// Name is an arbitrary name of the filter. Used for applying override
// settings in virtual host / route / weighted cluster configuration (not
// yet supported).
Name string
// Filter is the HTTP filter found in the registry for the config type.
Filter httpfilter.Builder
// Config contains the filter's configuration
Config httpfilter.FilterConfig
// Disabled specifies if the filter is disabled. For more information, see
// envoyproxy.io/docs/envoy/latest/intro/arch_overview/http/http_filters#route-based-filter-chain
Disabled bool
}
// InboundListenerConfig contains information about the inbound listener, i.e
// the server-side listener.
type InboundListenerConfig struct {
// Address is the local address on which the inbound listener is expected to
// accept incoming connections.
Address string
// Port is the local port on which the inbound listener is expected to
// accept incoming connections.
Port string
// DefaultFilterChain is the default filter chain to use if no other filter
// chain matches.
DefaultFilterChain NetworkFilterChainConfig
// FilterChains contains the filter chains associated with this listener.
FilterChains NetworkFilterChainMap
}

View File

@@ -0,0 +1,184 @@
/*
*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"regexp"
"time"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/internal/xds/clusterspecifier"
"google.golang.org/grpc/internal/xds/httpfilter"
"google.golang.org/grpc/internal/xds/matcher"
"google.golang.org/protobuf/types/known/anypb"
)
// RouteConfigUpdate contains information received in an RDS response, which is
// of interest to the registered RDS watcher.
type RouteConfigUpdate struct {
VirtualHosts []*VirtualHost
// ClusterSpecifierPlugins are the LB Configurations for any
// ClusterSpecifierPlugins referenced by the Route Table.
ClusterSpecifierPlugins map[string]clusterspecifier.BalancerConfig
// Raw is the resource from the xds response.
Raw *anypb.Any
}
// VirtualHost contains the routes for a list of Domains.
//
// Note that the domains in this slice can be a wildcard, not an exact string.
// The consumer of this struct needs to find the best match for its hostname.
type VirtualHost struct {
Domains []string
// Routes contains a list of routes, each containing matchers and
// corresponding action.
Routes []*Route
// HTTPFilterConfigOverride contains any HTTP filter config overrides for
// the virtual host which may be present. An individual filter's override
// may be unused if the matching Route contains an override for that
// filter.
HTTPFilterConfigOverride map[string]httpfilter.FilterConfig
RetryConfig *RetryConfig
}
// RetryConfig contains all retry-related configuration in either a VirtualHost
// or Route.
type RetryConfig struct {
// RetryOn is a set of status codes on which to retry. Only Canceled,
// DeadlineExceeded, Internal, ResourceExhausted, and Unavailable are
// supported; any other values will be omitted.
RetryOn map[codes.Code]bool
NumRetries uint32 // maximum number of retry attempts
RetryBackoff RetryBackoff // retry backoff policy
}
// RetryBackoff describes the backoff policy for retries.
type RetryBackoff struct {
BaseInterval time.Duration // initial backoff duration between attempts
MaxInterval time.Duration // maximum backoff duration
}
// HashPolicyType specifies the type of HashPolicy from a received RDS Response.
type HashPolicyType int
const (
// HashPolicyTypeHeader specifies to hash a Header in the incoming request.
HashPolicyTypeHeader HashPolicyType = iota
// HashPolicyTypeChannelID specifies to hash a unique Identifier of the
// Channel. This is a 64-bit random int computed at initialization time.
HashPolicyTypeChannelID
)
// HashPolicy specifies the HashPolicy if the upstream cluster uses a hashing
// load balancer.
type HashPolicy struct {
HashPolicyType HashPolicyType
Terminal bool
// Fields used for type HEADER.
HeaderName string
Regex *regexp.Regexp
RegexSubstitution string
}
// RouteActionType is the action of the route from a received RDS response.
type RouteActionType int
const (
// RouteActionUnsupported are routing types currently unsupported by grpc.
// According to A36, "A Route with an inappropriate action causes RPCs
// matching that route to fail."
RouteActionUnsupported RouteActionType = iota
// RouteActionRoute is the expected route type on the client side. Route
// represents routing a request to some upstream cluster. On the client
// side, if an RPC matches to a route that is not RouteActionRoute, the RPC
// will fail according to A36.
RouteActionRoute
// RouteActionNonForwardingAction is the expected route type on the server
// side. NonForwardingAction represents when a route will generate a
// response directly, without forwarding to an upstream host.
RouteActionNonForwardingAction
)
// Route is both a specification of how to match a request as well as an
// indication of the action to take upon match.
type Route struct {
Path *string
Prefix *string
Regex *regexp.Regexp
// Indicates if prefix/path matching should be case insensitive. The default
// is false (case sensitive).
CaseInsensitive bool
Headers []*HeaderMatcher
Fraction *uint32
HashPolicies []*HashPolicy
// If the matchers above indicate a match, the below configuration is used.
// If MaxStreamDuration is nil, it indicates neither of the route action's
// max_stream_duration fields (grpc_timeout_header_max nor
// max_stream_duration) were set. In this case, the ListenerUpdate's
// MaxStreamDuration field should be used. If MaxStreamDuration is set to
// an explicit zero duration, the application's deadline should be used.
MaxStreamDuration *time.Duration
// HTTPFilterConfigOverride contains any HTTP filter config overrides for
// the route which may be present. An individual filter's override may be
// unused if the matching WeightedCluster contains an override for that
// filter.
HTTPFilterConfigOverride map[string]httpfilter.FilterConfig
RetryConfig *RetryConfig
ActionType RouteActionType
// Only one of the following fields (WeightedClusters or
// ClusterSpecifierPlugin) will be set for a route.
WeightedClusters []WeightedCluster
// ClusterSpecifierPlugin is the name of the Cluster Specifier Plugin that
// this Route is linked to, if specified by xDS.
ClusterSpecifierPlugin string
// AutoHostRewrite indicates that the ":authority" header can be rewritten
// to the hostname of the upstream endpoint.
AutoHostRewrite bool
}
// WeightedCluster contains settings for an xds ActionType.WeightedCluster.
type WeightedCluster struct {
// Name is the name of the cluster.
Name string
// Weight is the relative weight of the cluster. It will never be zero.
Weight uint32
// HTTPFilterConfigOverride contains any HTTP filter config overrides for
// the weighted cluster which may be present.
HTTPFilterConfigOverride map[string]httpfilter.FilterConfig
}
// HeaderMatcher represents header matchers. Exact, prefix, suffix and contains
// matches are represented as a StringMatch.
type HeaderMatcher struct {
Name string
InvertMatch *bool
RegexMatch *regexp.Regexp
RangeMatch *Int64Range
PresentMatch *bool
StringMatch *matcher.StringMatcher
}
// Int64Range is a range for header range match.
type Int64Range struct {
Start int64
End int64
}

View File

@@ -0,0 +1,805 @@
/*
*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"encoding/json"
"errors"
"fmt"
"net"
"strconv"
"strings"
"time"
v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3"
v3http11proxypb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/http_11_proxy/v3"
v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
"google.golang.org/grpc/internal/envconfig"
"google.golang.org/grpc/internal/pretty"
iserviceconfig "google.golang.org/grpc/internal/serviceconfig"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/matcher"
"google.golang.org/grpc/internal/xds/xdsclient/xdslbregistry"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"google.golang.org/protobuf/types/known/structpb"
)
// ValidateClusterAndConstructClusterUpdateForTesting exports the
// validateClusterAndConstructClusterUpdate function for testing purposes.
var ValidateClusterAndConstructClusterUpdateForTesting = validateClusterAndConstructClusterUpdate
const (
maxSNILength = 255
// TransportSocket proto message has a `name` field which is expected to be set
// to this value by the management server.
transportSocketName = "envoy.transport_sockets.tls"
)
func unmarshalClusterResource(r *anypb.Any, serverCfg *bootstrap.ServerConfig) (string, ClusterUpdate, error) {
r, err := UnwrapResource(r)
if err != nil {
return "", ClusterUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err)
}
if !IsClusterResource(r.GetTypeUrl()) {
return "", ClusterUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl())
}
cluster := &v3clusterpb.Cluster{}
if err := proto.Unmarshal(r.GetValue(), cluster); err != nil {
return "", ClusterUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
}
if cluster.GetName() == "" {
return "", ClusterUpdate{}, fmt.Errorf("empty resource name in Cluster resource")
}
cu, err := validateClusterAndConstructClusterUpdate(cluster, serverCfg)
if err != nil {
return cluster.GetName(), ClusterUpdate{}, err
}
cu.Raw = r
return cluster.GetName(), cu, nil
}
const (
defaultRingHashMinSize = 1024
defaultRingHashMaxSize = 8 * 1024 * 1024 // 8M
ringHashSizeUpperBound = 8 * 1024 * 1024 // 8M
defaultLeastRequestChoiceCount = 2
)
func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster, serverCfg *bootstrap.ServerConfig) (ClusterUpdate, error) {
telemetryLabels := make(map[string]string)
if fmd := cluster.GetMetadata().GetFilterMetadata(); fmd != nil {
if val, ok := fmd["com.google.csm.telemetry_labels"]; ok {
if fields := val.GetFields(); fields != nil {
if val, ok := fields["service_name"]; ok {
if _, ok := val.GetKind().(*structpb.Value_StringValue); ok {
telemetryLabels["csm.service_name"] = val.GetStringValue()
}
}
if val, ok := fields["service_namespace"]; ok {
if _, ok := val.GetKind().(*structpb.Value_StringValue); ok {
telemetryLabels["csm.service_namespace_name"] = val.GetStringValue()
}
}
}
}
}
// "The values for the service labels csm.service_name and
// csm.service_namespace_name come from xDS, “unknown” if not present." -
// CSM Design.
if _, ok := telemetryLabels["csm.service_name"]; !ok {
telemetryLabels["csm.service_name"] = "unknown"
}
if _, ok := telemetryLabels["csm.service_namespace_name"]; !ok {
telemetryLabels["csm.service_namespace_name"] = "unknown"
}
var lbPolicy json.RawMessage
var err error
switch cluster.GetLbPolicy() {
case v3clusterpb.Cluster_ROUND_ROBIN:
lbPolicy = []byte(`[{"xds_wrr_locality_experimental": {"childPolicy": [{"round_robin": {}}]}}]`)
case v3clusterpb.Cluster_RING_HASH:
rhc := cluster.GetRingHashLbConfig()
if rhc.GetHashFunction() != v3clusterpb.Cluster_RingHashLbConfig_XX_HASH {
return ClusterUpdate{}, fmt.Errorf("unsupported ring_hash hash function %v in response: %+v", rhc.GetHashFunction(), cluster)
}
// Minimum defaults to 1024 entries, and limited to 8M entries Maximum
// defaults to 8M entries, and limited to 8M entries
var minSize, maxSize uint64 = defaultRingHashMinSize, defaultRingHashMaxSize
if min := rhc.GetMinimumRingSize(); min != nil {
minSize = min.GetValue()
}
if max := rhc.GetMaximumRingSize(); max != nil {
maxSize = max.GetValue()
}
rhLBCfg := []byte(fmt.Sprintf("{\"minRingSize\": %d, \"maxRingSize\": %d}", minSize, maxSize))
lbPolicy = []byte(fmt.Sprintf(`[{"ring_hash_experimental": %s}]`, rhLBCfg))
case v3clusterpb.Cluster_LEAST_REQUEST:
// "The configuration for the Least Request LB policy is the
// least_request_lb_config field. The field is optional; if not present,
// defaults will be assumed for all of its values." - A48
lr := cluster.GetLeastRequestLbConfig()
var choiceCount uint32 = defaultLeastRequestChoiceCount
if cc := lr.GetChoiceCount(); cc != nil {
choiceCount = cc.GetValue()
}
// "If choice_count < 2, the config will be rejected." - A48
if choiceCount < 2 {
return ClusterUpdate{}, fmt.Errorf("Cluster_LeastRequestLbConfig.ChoiceCount must be >= 2, got: %v", choiceCount)
}
lrLBCfg := []byte(fmt.Sprintf("{\"choiceCount\": %d}", choiceCount))
lbPolicy = []byte(fmt.Sprintf(`[{"least_request_experimental": %s}]`, lrLBCfg))
default:
return ClusterUpdate{}, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster)
}
// Process security configuration received from the control plane iff the
// corresponding environment variable is set.
var sc *SecurityConfig
var isHTTP11ProxyEnabled bool
if sc, isHTTP11ProxyEnabled, err = securityConfigFromCluster(cluster); err != nil {
return ClusterUpdate{}, err
}
// Process outlier detection received from the control plane iff the
// corresponding environment variable is set.
var od json.RawMessage
if od, err = outlierConfigFromCluster(cluster); err != nil {
return ClusterUpdate{}, err
}
if cluster.GetLoadBalancingPolicy() != nil {
lbPolicy, err = xdslbregistry.ConvertToServiceConfig(cluster.GetLoadBalancingPolicy(), 0)
if err != nil {
return ClusterUpdate{}, fmt.Errorf("error converting LoadBalancingPolicy %v in response: %+v: %v", cluster.GetLoadBalancingPolicy(), cluster, err)
}
// "It will be the responsibility of the XdsClient to validate the
// converted configuration. It will do this by having the gRPC LB policy
// registry parse the configuration." - A52
bc := &iserviceconfig.BalancerConfig{}
if err := json.Unmarshal(lbPolicy, bc); err != nil {
return ClusterUpdate{}, fmt.Errorf("JSON generated from xDS LB policy registry: %s is invalid: %v", pretty.FormatJSON(lbPolicy), err)
}
}
var lrsReportEndpointMetrics *LRSReportEndpointMetricsConfig
if envconfig.XDSORCAToLRSPropEnabled && len(cluster.GetLrsReportEndpointMetrics()) > 0 {
lrsReportEndpointMetrics = &LRSReportEndpointMetricsConfig{
NamedMetrics: make(map[string]struct{}),
}
for _, m := range cluster.GetLrsReportEndpointMetrics() {
switch m {
case "cpu_utilization":
lrsReportEndpointMetrics.CPUUtilization = true
case "mem_utilization":
lrsReportEndpointMetrics.MemUtilization = true
case "application_utilization":
lrsReportEndpointMetrics.ApplicationUtilization = true
case "named_metrics.*":
// If "named_metrics.*" is present, it takes precedence over any specific
// "named_metrics.foo" fields. Per gRFC A85, specific named metrics are ignored
// if NamedMetricsAll is true. We clear the map to save memory.
lrsReportEndpointMetrics.NamedMetricsAll = true
lrsReportEndpointMetrics.NamedMetrics = nil
default:
if lrsReportEndpointMetrics.NamedMetricsAll {
continue
}
if name, found := strings.CutPrefix(m, "named_metrics."); found && name != "" {
lrsReportEndpointMetrics.NamedMetrics[name] = struct{}{}
}
}
}
}
var metadata map[string]any
if envconfig.GCPAuthenticationFilterEnabled {
var err error
if metadata, err = validateAndConstructMetadata(cluster.GetMetadata()); err != nil {
return ClusterUpdate{}, err
}
}
ret := ClusterUpdate{
ClusterName: cluster.GetName(),
SecurityCfg: sc,
MaxRequests: circuitBreakersFromCluster(cluster),
LBPolicy: lbPolicy,
OutlierDetection: od,
TelemetryLabels: telemetryLabels,
LRSReportEndpointMetrics: lrsReportEndpointMetrics,
Metadata: metadata,
IsHTTP11ProxyEnabled: isHTTP11ProxyEnabled,
}
if lrs := cluster.GetLrsServer(); lrs != nil {
if lrs.GetSelf() == nil {
return ClusterUpdate{}, fmt.Errorf("unsupported config_source_specifier %T in lrs_server field", lrs.ConfigSourceSpecifier)
}
ret.LRSServerConfig = serverCfg
}
// Validate and set cluster type from the response.
switch {
case cluster.GetType() == v3clusterpb.Cluster_EDS:
if configsource := cluster.GetEdsClusterConfig().GetEdsConfig(); configsource.GetAds() == nil && configsource.GetSelf() == nil {
return ClusterUpdate{}, fmt.Errorf("CDS's EDS config source is not ADS or Self: %+v", cluster)
}
ret.ClusterType = ClusterTypeEDS
ret.EDSServiceName = cluster.GetEdsClusterConfig().GetServiceName()
if strings.HasPrefix(ret.ClusterName, "xdstp:") && ret.EDSServiceName == "" {
return ClusterUpdate{}, fmt.Errorf("CDS's EDS service name is not set with a new-style cluster name: %+v", cluster)
}
return ret, nil
case cluster.GetType() == v3clusterpb.Cluster_LOGICAL_DNS:
ret.ClusterType = ClusterTypeLogicalDNS
dnsHN, err := dnsHostNameFromCluster(cluster)
if err != nil {
return ClusterUpdate{}, err
}
ret.DNSHostName = dnsHN
return ret, nil
case cluster.GetClusterType() != nil && cluster.GetClusterType().Name == "envoy.clusters.aggregate":
clusters := &v3aggregateclusterpb.ClusterConfig{}
if err := proto.Unmarshal(cluster.GetClusterType().GetTypedConfig().GetValue(), clusters); err != nil {
return ClusterUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
}
if len(clusters.Clusters) == 0 {
return ClusterUpdate{}, fmt.Errorf("xds: aggregate cluster has empty clusters field in response: %+v", cluster)
}
ret.ClusterType = ClusterTypeAggregate
ret.PrioritizedClusterNames = clusters.Clusters
return ret, nil
default:
return ClusterUpdate{}, fmt.Errorf("unsupported cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster)
}
}
// dnsHostNameFromCluster extracts the DNS host name from the cluster's load
// assignment.
//
// There should be exactly one locality, with one endpoint, whose address
// contains the address and port.
func dnsHostNameFromCluster(cluster *v3clusterpb.Cluster) (string, error) {
loadAssignment := cluster.GetLoadAssignment()
if loadAssignment == nil {
return "", fmt.Errorf("load_assignment not present for LOGICAL_DNS cluster")
}
if len(loadAssignment.GetEndpoints()) != 1 {
return "", fmt.Errorf("load_assignment for LOGICAL_DNS cluster must have exactly one locality, got: %+v", loadAssignment)
}
endpoints := loadAssignment.GetEndpoints()[0].GetLbEndpoints()
if len(endpoints) != 1 {
return "", fmt.Errorf("locality for LOGICAL_DNS cluster must have exactly one endpoint, got: %+v", endpoints)
}
endpoint := endpoints[0].GetEndpoint()
if endpoint == nil {
return "", fmt.Errorf("endpoint for LOGICAL_DNS cluster not set")
}
socketAddr := endpoint.GetAddress().GetSocketAddress()
if socketAddr == nil {
return "", fmt.Errorf("socket address for endpoint for LOGICAL_DNS cluster not set")
}
if socketAddr.GetResolverName() != "" {
return "", fmt.Errorf("socket address for endpoint for LOGICAL_DNS cluster not set has unexpected custom resolver name: %v", socketAddr.GetResolverName())
}
host := socketAddr.GetAddress()
if host == "" {
return "", fmt.Errorf("host for endpoint for LOGICAL_DNS cluster not set")
}
port := socketAddr.GetPortValue()
if port == 0 {
return "", fmt.Errorf("port for endpoint for LOGICAL_DNS cluster not set")
}
return net.JoinHostPort(host, strconv.Itoa(int(port))), nil
}
// securityConfigFromCluster extracts the relevant security configuration from
// the received Cluster resource.
func securityConfigFromCluster(cluster *v3clusterpb.Cluster) (*SecurityConfig, bool, error) {
if tsm := cluster.GetTransportSocketMatches(); len(tsm) != 0 {
return nil, false, fmt.Errorf("unsupported transport_socket_matches field is non-empty: %+v", tsm)
}
// The Cluster resource contains a `transport_socket` field, which contains
// a oneof `typed_config` field of type `protobuf.Any`. The any proto
// contains a marshaled representation of an `UpstreamTlsContext` message.
ts := cluster.GetTransportSocket()
if ts == nil {
return nil, false, nil
}
var isHTTP11ProxyEnabled bool
tc := ts.GetTypedConfig()
typeURL := tc.GetTypeUrl()
if envconfig.XDSHTTPConnectEnabled && typeURL == version.V3HTTP11ProxyUpstreamTransportURL {
isHTTP11ProxyEnabled = true
http11Proxy := &v3http11proxypb.Http11ProxyUpstreamTransport{}
if err := proto.Unmarshal(tc.GetValue(), http11Proxy); err != nil {
return nil, false, fmt.Errorf("failed to unmarshal Http11ProxyUpstreamTransport in CDS response: %v", err)
}
ts = http11Proxy.GetTransportSocket()
if ts == nil {
return nil, true, nil
}
tc = ts.GetTypedConfig()
typeURL = tc.GetTypeUrl()
}
if name := ts.GetName(); name != transportSocketName {
return nil, false, fmt.Errorf("transport_socket field has unexpected name: %s", name)
}
if typeURL != version.V3UpstreamTLSContextURL {
return nil, false, fmt.Errorf("transport_socket missing typed_config or wrong type_url: %q", typeURL)
}
upstreamCtx := &v3tlspb.UpstreamTlsContext{}
if err := proto.Unmarshal(tc.GetValue(), upstreamCtx); err != nil {
return nil, false, fmt.Errorf("failed to unmarshal UpstreamTlsContext in CDS response: %v", err)
}
// The following fields from `UpstreamTlsContext` are ignored:
// - allow_renegotiation
// - max_session_keys
if upstreamCtx.GetCommonTlsContext() == nil {
return nil, false, errors.New("UpstreamTlsContext in CDS response does not contain a CommonTlsContext")
}
sc, err := securityConfigFromCommonTLSContext(upstreamCtx.GetCommonTlsContext(), false)
if err != nil {
return nil, false, err
}
// Set SNI related fields in SecurityConfig from UpstreamTlsContext if
// `GRPC_EXPERIMENTAL_XDS_SNI` is enabled.
if envconfig.XDSSNIEnabled {
sc.SNI = upstreamCtx.GetSni()
if len(sc.SNI) > maxSNILength {
return nil, false, fmt.Errorf("SNI value %q in UpstreamTlsContext in CDS response exceeds max length of %d", sc.SNI, maxSNILength)
}
sc.UseAutoHostSNI = upstreamCtx.GetAutoHostSni()
sc.AutoSNISANValidation = upstreamCtx.GetAutoSniSanValidation()
}
return sc, isHTTP11ProxyEnabled, nil
}
// common is expected to be not nil.
// The `alpn_protocols` field is ignored.
func securityConfigFromCommonTLSContext(common *v3tlspb.CommonTlsContext, server bool) (*SecurityConfig, error) {
if common.GetTlsParams() != nil {
return nil, fmt.Errorf("unsupported tls_params field in CommonTlsContext message: %+v", common)
}
if common.GetCustomHandshaker() != nil {
return nil, fmt.Errorf("unsupported custom_handshaker field in CommonTlsContext message: %+v", common)
}
// For now, if we can't get a valid security config from the new fields, we
// fallback to the old deprecated fields.
// TODO: Drop support for deprecated fields. NACK if err != nil here.
sc, err1 := securityConfigFromCommonTLSContextUsingNewFields(common, server)
if sc == nil || sc.Equal(&SecurityConfig{}) {
var err error
sc, err = securityConfigFromCommonTLSContextWithDeprecatedFields(common, server)
if err != nil {
// Retain the validation error from using the new fields.
return nil, errors.Join(err1, fmt.Errorf("failed to parse config using deprecated fields: %v", err))
}
}
if sc != nil {
// sc == nil is a valid case where the control plane has not sent us any
// security configuration. xDS creds will use fallback creds.
if server {
if sc.IdentityInstanceName == "" {
return nil, errors.New("security configuration on the server-side does not contain identity certificate provider instance name")
}
} else {
if !sc.UseSystemRootCerts && sc.RootInstanceName == "" {
return nil, errors.New("security configuration on the client-side does not contain root certificate provider instance name")
}
}
}
return sc, nil
}
func securityConfigFromCommonTLSContextWithDeprecatedFields(common *v3tlspb.CommonTlsContext, server bool) (*SecurityConfig, error) {
// The `CommonTlsContext` contains a
// `tls_certificate_certificate_provider_instance` field of type
// `CertificateProviderInstance`, which contains the provider instance name
// and the certificate name to fetch identity certs.
sc := &SecurityConfig{}
if identity := common.GetTlsCertificateCertificateProviderInstance(); identity != nil {
sc.IdentityInstanceName = identity.GetInstanceName()
sc.IdentityCertName = identity.GetCertificateName()
}
// The `CommonTlsContext` contains a `validation_context_type` field which
// is a oneof. We can get the values that we are interested in from two of
// those possible values:
// - combined validation context:
// - contains a default validation context which holds the list of
// matchers for accepted SANs.
// - contains certificate provider instance configuration
// - certificate provider instance configuration
// - in this case, we do not get a list of accepted SANs.
switch t := common.GetValidationContextType().(type) {
case *v3tlspb.CommonTlsContext_CombinedValidationContext:
combined := common.GetCombinedValidationContext()
var matchers []matcher.StringMatcher
if def := combined.GetDefaultValidationContext(); def != nil {
for _, m := range def.GetMatchSubjectAltNames() {
matcher, err := matcher.StringMatcherFromProto(m)
if err != nil {
return nil, err
}
matchers = append(matchers, matcher)
}
}
if server && len(matchers) != 0 {
return nil, fmt.Errorf("match_subject_alt_names field in validation context is not supported on the server: %v", common)
}
sc.SubjectAltNameMatchers = matchers
if pi := combined.GetValidationContextCertificateProviderInstance(); pi != nil {
sc.RootInstanceName = pi.GetInstanceName()
sc.RootCertName = pi.GetCertificateName()
}
case *v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance:
pi := common.GetValidationContextCertificateProviderInstance()
sc.RootInstanceName = pi.GetInstanceName()
sc.RootCertName = pi.GetCertificateName()
case nil:
// It is valid for the validation context to be nil on the server side.
default:
return nil, fmt.Errorf("validation context contains unexpected type: %T", t)
}
return sc, nil
}
// gRFC A29 https://github.com/grpc/proposal/blob/master/A29-xds-tls-security.md
// specifies the new way to fetch security configuration and says the following:
//
// Although there are various ways to obtain certificates as per this proto
// (which are supported by Envoy), gRPC supports only one of them and that is
// the `CertificateProviderPluginInstance` proto.
//
// This helper function attempts to fetch security configuration from the
// `CertificateProviderPluginInstance` message, given a CommonTlsContext.
func securityConfigFromCommonTLSContextUsingNewFields(common *v3tlspb.CommonTlsContext, server bool) (*SecurityConfig, error) {
// The `tls_certificate_provider_instance` field of type
// `CertificateProviderPluginInstance` is used to fetch the identity
// certificate provider.
sc := &SecurityConfig{}
identity := common.GetTlsCertificateProviderInstance()
if identity == nil && len(common.GetTlsCertificates()) != 0 {
return nil, fmt.Errorf("expected field tls_certificate_provider_instance is not set, while unsupported field tls_certificates is set in CommonTlsContext message: %+v", common)
}
if identity == nil && common.GetTlsCertificateSdsSecretConfigs() != nil {
return nil, fmt.Errorf("expected field tls_certificate_provider_instance is not set, while unsupported field tls_certificate_sds_secret_configs is set in CommonTlsContext message: %+v", common)
}
sc.IdentityInstanceName = identity.GetInstanceName()
sc.IdentityCertName = identity.GetCertificateName()
// The `CommonTlsContext` contains a oneof field `validation_context_type`,
// which contains the `CertificateValidationContext` message in one of the
// following ways:
// - `validation_context` field
// - this is directly of type `CertificateValidationContext`
// - `combined_validation_context` field
// - this is of type `CombinedCertificateValidationContext` and contains
// a `default validation context` field of type
// `CertificateValidationContext`
//
// The `CertificateValidationContext` message has the following fields that
// we are interested in:
// - `ca_certificate_provider_instance`
// - this is of type `CertificateProviderPluginInstance`
// - `system_root_certs`:
// - This indicates the usage of system root certs for validation.
// - `match_subject_alt_names`
// - this is a list of string matchers
//
// The `CertificateProviderPluginInstance` message contains two fields
// - instance_name
// - this is the certificate provider instance name to be looked up in
// the bootstrap configuration
// - certificate_name
// - this is an opaque name passed to the certificate provider
var validationCtx *v3tlspb.CertificateValidationContext
switch typ := common.GetValidationContextType().(type) {
case *v3tlspb.CommonTlsContext_ValidationContext:
validationCtx = common.GetValidationContext()
case *v3tlspb.CommonTlsContext_CombinedValidationContext:
validationCtx = common.GetCombinedValidationContext().GetDefaultValidationContext()
case nil:
// It is valid for the validation context to be nil on the server side.
return sc, nil
default:
return nil, fmt.Errorf("validation context contains unexpected type: %T", typ)
}
// If we get here, it means that the `CertificateValidationContext` message
// was found through one of the supported ways. It is an error if the
// validation context is specified, but it does not specify a way to
// validate TLS certificates. Peer TLS certs can be verified in the
// following ways:
// 1. If the ca_certificate_provider_instance field is set, it contains
// information about the certificate provider to be used for the root
// certificates, else
// 2. If the system_root_certs field is set, and the config is for a client,
// use the system default root certs.
useSystemRootCerts := false
if validationCtx.GetCaCertificateProviderInstance() == nil && envconfig.XDSSystemRootCertsEnabled {
if server {
if validationCtx.GetSystemRootCerts() != nil {
// The `system_root_certs` field will not be supported on the
// gRPC server side. If `ca_certificate_provider_instance` is
// unset and `system_root_certs` is set, the LDS resource will
// be NACKed.
// - A82
return nil, fmt.Errorf("expected field ca_certificate_provider_instance is missing and unexpected field system_root_certs is set for server in CommonTlsContext message: %+v", common)
}
} else {
if validationCtx.GetSystemRootCerts() != nil {
useSystemRootCerts = true
}
}
}
// The following fields are ignored:
// - trusted_ca
// - watched_directory
// - allow_expired_certificate
// - trust_chain_verification
switch {
case len(validationCtx.GetVerifyCertificateSpki()) != 0:
return nil, fmt.Errorf("unsupported verify_certificate_spki field in CommonTlsContext message: %+v", common)
case len(validationCtx.GetVerifyCertificateHash()) != 0:
return nil, fmt.Errorf("unsupported verify_certificate_hash field in CommonTlsContext message: %+v", common)
case validationCtx.GetRequireSignedCertificateTimestamp().GetValue():
return nil, fmt.Errorf("unsupported require_signed_certificate_timestamp field in CommonTlsContext message: %+v", common)
case validationCtx.GetCrl() != nil:
return nil, fmt.Errorf("unsupported crl field in CommonTlsContext message: %+v", common)
case validationCtx.GetCustomValidatorConfig() != nil:
return nil, fmt.Errorf("unsupported custom_validator_config field in CommonTlsContext message: %+v", common)
}
if rootProvider := validationCtx.GetCaCertificateProviderInstance(); rootProvider != nil {
sc.RootInstanceName = rootProvider.GetInstanceName()
sc.RootCertName = rootProvider.GetCertificateName()
} else if useSystemRootCerts {
sc.UseSystemRootCerts = true
} else if !server && envconfig.XDSSystemRootCertsEnabled {
return nil, fmt.Errorf("expected fields ca_certificate_provider_instance and system_root_certs are missing in CommonTlsContext message: %+v", common)
} else {
// Don't mention the system_root_certs field if it was not checked.
return nil, fmt.Errorf("expected field ca_certificate_provider_instance is missing in CommonTlsContext message: %+v", common)
}
var matchers []matcher.StringMatcher
for _, m := range validationCtx.GetMatchSubjectAltNames() {
matcher, err := matcher.StringMatcherFromProto(m)
if err != nil {
return nil, err
}
matchers = append(matchers, matcher)
}
if server && len(matchers) != 0 {
return nil, fmt.Errorf("match_subject_alt_names field in validation context is not supported on the server: %v", common)
}
sc.SubjectAltNameMatchers = matchers
return sc, nil
}
// circuitBreakersFromCluster extracts the circuit breakers configuration from
// the received cluster resource. Returns nil if no CircuitBreakers or no
// Thresholds in CircuitBreakers.
func circuitBreakersFromCluster(cluster *v3clusterpb.Cluster) *uint32 {
for _, threshold := range cluster.GetCircuitBreakers().GetThresholds() {
if threshold.GetPriority() != v3corepb.RoutingPriority_DEFAULT {
continue
}
maxRequestsPb := threshold.GetMaxRequests()
if maxRequestsPb == nil {
return nil
}
maxRequests := maxRequestsPb.GetValue()
return &maxRequests
}
return nil
}
// idurationp takes a time.Duration and converts it to an internal duration, and
// returns a pointer to that internal duration.
func idurationp(d time.Duration) *iserviceconfig.Duration {
id := iserviceconfig.Duration(d)
return &id
}
func uint32p(i uint32) *uint32 {
return &i
}
// Helper types to prepare Outlier Detection JSON. Pointer types to distinguish
// between unset and a zero value.
type successRateEjection struct {
StdevFactor *uint32 `json:"stdevFactor,omitempty"`
EnforcementPercentage *uint32 `json:"enforcementPercentage,omitempty"`
MinimumHosts *uint32 `json:"minimumHosts,omitempty"`
RequestVolume *uint32 `json:"requestVolume,omitempty"`
}
type failurePercentageEjection struct {
Threshold *uint32 `json:"threshold,omitempty"`
EnforcementPercentage *uint32 `json:"enforcementPercentage,omitempty"`
MinimumHosts *uint32 `json:"minimumHosts,omitempty"`
RequestVolume *uint32 `json:"requestVolume,omitempty"`
}
type odLBConfig struct {
Interval *iserviceconfig.Duration `json:"interval,omitempty"`
BaseEjectionTime *iserviceconfig.Duration `json:"baseEjectionTime,omitempty"`
MaxEjectionTime *iserviceconfig.Duration `json:"maxEjectionTime,omitempty"`
MaxEjectionPercent *uint32 `json:"maxEjectionPercent,omitempty"`
SuccessRateEjection *successRateEjection `json:"successRateEjection,omitempty"`
FailurePercentageEjection *failurePercentageEjection `json:"failurePercentageEjection,omitempty"`
}
// outlierConfigFromCluster converts the received Outlier Detection
// configuration into JSON configuration for Outlier Detection, taking into
// account xDS Defaults. Returns nil if no OutlierDetection field set in the
// cluster resource.
func outlierConfigFromCluster(cluster *v3clusterpb.Cluster) (json.RawMessage, error) {
od := cluster.GetOutlierDetection()
if od == nil {
return nil, nil
}
// "The outlier_detection field of the Cluster resource should have its fields
// validated according to the rules for the corresponding LB policy config
// fields in the above "Validation" section. If any of these requirements is
// violated, the Cluster resource should be NACKed." - A50
// "The google.protobuf.Duration fields interval, base_ejection_time, and
// max_ejection_time must obey the restrictions in the
// google.protobuf.Duration documentation and they must have non-negative
// values." - A50
var interval *iserviceconfig.Duration
if i := od.GetInterval(); i != nil {
if err := i.CheckValid(); err != nil {
return nil, fmt.Errorf("outlier_detection.interval is invalid with error: %v", err)
}
if interval = idurationp(i.AsDuration()); *interval < 0 {
return nil, fmt.Errorf("outlier_detection.interval = %v; must be a valid duration and >= 0", *interval)
}
}
var baseEjectionTime *iserviceconfig.Duration
if bet := od.GetBaseEjectionTime(); bet != nil {
if err := bet.CheckValid(); err != nil {
return nil, fmt.Errorf("outlier_detection.base_ejection_time is invalid with error: %v", err)
}
if baseEjectionTime = idurationp(bet.AsDuration()); *baseEjectionTime < 0 {
return nil, fmt.Errorf("outlier_detection.base_ejection_time = %v; must be >= 0", *baseEjectionTime)
}
}
var maxEjectionTime *iserviceconfig.Duration
if met := od.GetMaxEjectionTime(); met != nil {
if err := met.CheckValid(); err != nil {
return nil, fmt.Errorf("outlier_detection.max_ejection_time is invalid: %v", err)
}
if maxEjectionTime = idurationp(met.AsDuration()); *maxEjectionTime < 0 {
return nil, fmt.Errorf("outlier_detection.max_ejection_time = %v; must be >= 0", *maxEjectionTime)
}
}
// "The fields max_ejection_percent, enforcing_success_rate,
// failure_percentage_threshold, and enforcing_failure_percentage must have
// values less than or equal to 100. If any of these requirements is
// violated, the Cluster resource should be NACKed." - A50
var maxEjectionPercent *uint32
if mep := od.GetMaxEjectionPercent(); mep != nil {
if maxEjectionPercent = uint32p(mep.GetValue()); *maxEjectionPercent > 100 {
return nil, fmt.Errorf("outlier_detection.max_ejection_percent = %v; must be <= 100", *maxEjectionPercent)
}
}
// "if the enforcing_success_rate field is set to 0, the config
// success_rate_ejection field will be null and all success_rate_* fields
// will be ignored." - A50
var enforcingSuccessRate *uint32
if esr := od.GetEnforcingSuccessRate(); esr != nil {
if enforcingSuccessRate = uint32p(esr.GetValue()); *enforcingSuccessRate > 100 {
return nil, fmt.Errorf("outlier_detection.enforcing_success_rate = %v; must be <= 100", *enforcingSuccessRate)
}
}
var failurePercentageThreshold *uint32
if fpt := od.GetFailurePercentageThreshold(); fpt != nil {
if failurePercentageThreshold = uint32p(fpt.GetValue()); *failurePercentageThreshold > 100 {
return nil, fmt.Errorf("outlier_detection.failure_percentage_threshold = %v; must be <= 100", *failurePercentageThreshold)
}
}
// "If the enforcing_failure_percent field is set to 0 or null, the config
// failure_percent_ejection field will be null and all failure_percent_*
// fields will be ignored." - A50
var enforcingFailurePercentage *uint32
if efp := od.GetEnforcingFailurePercentage(); efp != nil {
if enforcingFailurePercentage = uint32p(efp.GetValue()); *enforcingFailurePercentage > 100 {
return nil, fmt.Errorf("outlier_detection.enforcing_failure_percentage = %v; must be <= 100", *enforcingFailurePercentage)
}
}
var successRateStdevFactor *uint32
if srsf := od.GetSuccessRateStdevFactor(); srsf != nil {
successRateStdevFactor = uint32p(srsf.GetValue())
}
var successRateMinimumHosts *uint32
if srmh := od.GetSuccessRateMinimumHosts(); srmh != nil {
successRateMinimumHosts = uint32p(srmh.GetValue())
}
var successRateRequestVolume *uint32
if srrv := od.GetSuccessRateRequestVolume(); srrv != nil {
successRateRequestVolume = uint32p(srrv.GetValue())
}
var failurePercentageMinimumHosts *uint32
if fpmh := od.GetFailurePercentageMinimumHosts(); fpmh != nil {
failurePercentageMinimumHosts = uint32p(fpmh.GetValue())
}
var failurePercentageRequestVolume *uint32
if fprv := od.GetFailurePercentageRequestVolume(); fprv != nil {
failurePercentageRequestVolume = uint32p(fprv.GetValue())
}
// "if the enforcing_success_rate field is set to 0, the config
// success_rate_ejection field will be null and all success_rate_* fields
// will be ignored." - A50
var sre *successRateEjection
if enforcingSuccessRate == nil || *enforcingSuccessRate != 0 {
sre = &successRateEjection{
StdevFactor: successRateStdevFactor,
EnforcementPercentage: enforcingSuccessRate,
MinimumHosts: successRateMinimumHosts,
RequestVolume: successRateRequestVolume,
}
}
// "If the enforcing_failure_percent field is set to 0 or null, the config
// failure_percent_ejection field will be null and all failure_percent_*
// fields will be ignored." - A50
var fpe *failurePercentageEjection
if enforcingFailurePercentage != nil && *enforcingFailurePercentage != 0 {
fpe = &failurePercentageEjection{
Threshold: failurePercentageThreshold,
EnforcementPercentage: enforcingFailurePercentage,
MinimumHosts: failurePercentageMinimumHosts,
RequestVolume: failurePercentageRequestVolume,
}
}
odLBCfg := &odLBConfig{
Interval: interval,
BaseEjectionTime: baseEjectionTime,
MaxEjectionTime: maxEjectionTime,
MaxEjectionPercent: maxEjectionPercent,
SuccessRateEjection: sre,
FailurePercentageEjection: fpe,
}
return json.Marshal(odLBCfg)
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,294 @@
/*
*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"fmt"
"math"
"net"
"strconv"
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
"google.golang.org/grpc/experimental/balancer/hostname"
"google.golang.org/grpc/internal/envconfig"
"google.golang.org/grpc/internal/pretty"
xdsinternal "google.golang.org/grpc/internal/xds"
"google.golang.org/grpc/internal/xds/clients"
"google.golang.org/grpc/resolver"
"google.golang.org/grpc/resolver/ringhash"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
)
// Hostname returns the hostname from the BalancerAttributes of the
// given Address. If this attribute is not set, it returns the empty
// string.
func Hostname(addr resolver.Address) string {
ep := resolver.Endpoint{Attributes: addr.BalancerAttributes}
return hostname.FromEndpoint(ep)
}
func unmarshalEndpointsResource(r *anypb.Any) (string, EndpointsUpdate, error) {
r, err := UnwrapResource(r)
if err != nil {
return "", EndpointsUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err)
}
if !IsEndpointsResource(r.GetTypeUrl()) {
return "", EndpointsUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl())
}
cla := &v3endpointpb.ClusterLoadAssignment{}
if err := proto.Unmarshal(r.GetValue(), cla); err != nil {
return "", EndpointsUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
}
if cla.GetClusterName() == "" {
return "", EndpointsUpdate{}, fmt.Errorf("empty resource name in endpoints resource")
}
u, err := parseEDSRespProto(cla)
if err != nil {
return cla.GetClusterName(), EndpointsUpdate{}, err
}
u.Raw = r
return cla.GetClusterName(), u, nil
}
func parseAddress(socketAddress *v3corepb.SocketAddress) string {
return net.JoinHostPort(socketAddress.GetAddress(), strconv.Itoa(int(socketAddress.GetPortValue())))
}
func parseDropPolicy(dropPolicy *v3endpointpb.ClusterLoadAssignment_Policy_DropOverload) (OverloadDropConfig, error) {
percentage := dropPolicy.GetDropPercentage()
var (
numerator = percentage.GetNumerator()
denominator uint32
)
switch percentage.GetDenominator() {
case v3typepb.FractionalPercent_HUNDRED:
denominator = 100
case v3typepb.FractionalPercent_TEN_THOUSAND:
denominator = 10000
case v3typepb.FractionalPercent_MILLION:
denominator = 1000000
default:
return OverloadDropConfig{}, fmt.Errorf("EDS response contains a drop policy with unsupported denominator: %v", percentage.GetDenominator())
}
return OverloadDropConfig{
Category: dropPolicy.GetCategory(),
Numerator: numerator,
Denominator: denominator,
}, nil
}
func parseEndpoints(lbEndpoints []*v3endpointpb.LbEndpoint, uniqueEndpointAddrs map[string]bool) ([]Endpoint, error) {
endpoints := make([]Endpoint, 0, len(lbEndpoints))
var totalWeight uint64
for _, lbEndpoint := range lbEndpoints {
// If the load_balancing_weight field is specified, it must be set to a
// value of at least 1. If unspecified, each host is presumed to have
// equal weight in a locality.
weight := uint32(1)
if w := lbEndpoint.GetLoadBalancingWeight(); w != nil {
if w.GetValue() == 0 {
return nil, fmt.Errorf("EDS response contains an endpoint with zero weight: %+v", lbEndpoint)
}
weight = w.GetValue()
}
totalWeight += uint64(weight)
if totalWeight > math.MaxUint32 {
return nil, fmt.Errorf("sum of weights of endpoints in the same locality exceeds maximum value %d", uint64(math.MaxUint32))
}
addrs := []string{parseAddress(lbEndpoint.GetEndpoint().GetAddress().GetSocketAddress())}
if envconfig.XDSDualstackEndpointsEnabled {
for _, sa := range lbEndpoint.GetEndpoint().GetAdditionalAddresses() {
addrs = append(addrs, parseAddress(sa.GetAddress().GetSocketAddress()))
}
}
address := []resolver.Address{}
for _, a := range addrs {
address = append(address, resolver.Address{Addr: a})
if uniqueEndpointAddrs[a] {
return nil, fmt.Errorf("duplicate endpoint with the same address %s", a)
}
uniqueEndpointAddrs[a] = true
}
var endpointMetadata map[string]any
var hashKey string
if envconfig.XDSHTTPConnectEnabled || !envconfig.XDSEndpointHashKeyBackwardCompat {
var err error
endpointMetadata, err = validateAndConstructMetadata(lbEndpoint.GetMetadata())
if err != nil {
return nil, err
}
// "The xDS resolver, described in A74, will be changed to set the hash_key
// endpoint attribute to the value of LbEndpoint.Metadata envoy.lb hash_key
// field, as described in Envoy's documentation for the ring hash load
// balancer." - A76
if !envconfig.XDSEndpointHashKeyBackwardCompat {
hashKey = hashKeyFromMetadata(endpointMetadata)
}
}
endpoint := resolver.Endpoint{Addresses: address}
endpoint = hostname.Set(endpoint, lbEndpoint.GetEndpoint().GetHostname())
endpoint = ringhash.SetHashKey(endpoint, hashKey)
endpoints = append(endpoints, Endpoint{
ResolverEndpoint: endpoint,
HealthStatus: EndpointHealthStatus(lbEndpoint.GetHealthStatus()),
Weight: weight,
Metadata: endpointMetadata,
})
}
return endpoints, nil
}
// hashKey extracts and returns the hash key from the given endpoint metadata.
// If no hash key is found, it returns an empty string.
func hashKeyFromMetadata(metadata map[string]any) string {
envoyLB, ok := metadata["envoy.lb"].(StructMetadataValue)
if ok {
if h, ok := envoyLB.Data["hash_key"].(string); ok {
return h
}
}
return ""
}
func parseEDSRespProto(m *v3endpointpb.ClusterLoadAssignment) (EndpointsUpdate, error) {
ret := EndpointsUpdate{}
for _, dropPolicy := range m.GetPolicy().GetDropOverloads() {
drop, err := parseDropPolicy(dropPolicy)
if err != nil {
return EndpointsUpdate{}, err
}
ret.Drops = append(ret.Drops, drop)
}
priorities := make(map[uint32]map[string]bool)
sumOfWeights := make(map[uint32]uint64)
uniqueEndpointAddrs := make(map[string]bool)
for _, locality := range m.Endpoints {
l := locality.GetLocality()
if l == nil {
return EndpointsUpdate{}, fmt.Errorf("EDS response contains a locality without ID, locality: %+v", locality)
}
weight := locality.GetLoadBalancingWeight().GetValue()
if weight == 0 {
logger.Warningf("Ignoring locality %s with weight 0", pretty.ToJSON(l))
continue
}
priority := locality.GetPriority()
sumOfWeights[priority] += uint64(weight)
if sumOfWeights[priority] > math.MaxUint32 {
return EndpointsUpdate{}, fmt.Errorf("sum of weights of localities at the same priority %d exceeded maximal value", priority)
}
localitiesWithPriority := priorities[priority]
if localitiesWithPriority == nil {
localitiesWithPriority = make(map[string]bool)
priorities[priority] = localitiesWithPriority
}
lid := clients.Locality{
Region: l.Region,
Zone: l.Zone,
SubZone: l.SubZone,
}
lidStr := xdsinternal.LocalityString(lid)
// "Since an xDS configuration can place a given locality under multiple
// priorities, it is possible to see locality weight attributes with
// different values for the same locality." - A52
//
// This is handled in the client by emitting the locality weight
// specified for the priority it is specified in. If the same locality
// has a different weight in two priorities, each priority will specify
// a locality with the locality weight specified for that priority, and
// thus the subsequent tree of balancers linked to that priority will
// use that locality weight as well.
if localitiesWithPriority[lidStr] {
return EndpointsUpdate{}, fmt.Errorf("duplicate locality %s with the same priority %v", lidStr, priority)
}
localitiesWithPriority[lidStr] = true
endpoints, err := parseEndpoints(locality.GetLbEndpoints(), uniqueEndpointAddrs)
if err != nil {
return EndpointsUpdate{}, err
}
var localityMetadata map[string]any
if envconfig.XDSHTTPConnectEnabled {
var err error
localityMetadata, err = validateAndConstructMetadata(locality.GetMetadata())
if err != nil {
return EndpointsUpdate{}, err
}
}
ret.Localities = append(ret.Localities, Locality{
ID: lid,
Endpoints: endpoints,
Weight: weight,
Priority: priority,
Metadata: localityMetadata,
})
}
for i := 0; i < len(priorities); i++ {
if _, ok := priorities[uint32(i)]; !ok {
return EndpointsUpdate{}, fmt.Errorf("priority %v missing (with different priorities %v received)", i, priorities)
}
}
return ret, nil
}
// validateAndConstructMetadata processes the metadata from the xDS resource
// and returns a map of parsed metadata values.
func validateAndConstructMetadata(metadataProto *v3corepb.Metadata) (map[string]any, error) {
if metadataProto == nil {
return nil, nil
}
metadata := make(map[string]any)
// First go through TypedFilterMetadata.
for key, anyProto := range metadataProto.GetTypedFilterMetadata() {
converter := metadataConverterForType(anyProto.GetTypeUrl())
// Ignore types we don't have a converter for.
if converter == nil {
continue
}
val, err := converter.convert(anyProto)
if err != nil {
// If the converter fails, nack the whole resource.
return nil, fmt.Errorf("metadata conversion for key %q and type %q failed: %v", key, anyProto.GetTypeUrl(), err)
}
metadata[key] = val
}
// Process FilterMetadata for any keys not already handled.
for key, structProto := range metadataProto.GetFilterMetadata() {
// Skip keys already added from TypedFilterMetadata.
if metadata[key] != nil {
continue
}
metadata[key] = StructMetadataValue{Data: structProto.AsMap()}
}
return metadata, nil
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,618 @@
/*
*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"errors"
"fmt"
"net/netip"
"strconv"
v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1"
v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
"google.golang.org/grpc/internal/envconfig"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/httpfilter"
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
)
func unmarshalListenerResource(r *anypb.Any, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (string, ListenerUpdate, error) {
r, err := UnwrapResource(r)
if err != nil {
return "", ListenerUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err)
}
if !IsListenerResource(r.GetTypeUrl()) {
return "", ListenerUpdate{}, fmt.Errorf("unexpected listener resource type: %q ", r.GetTypeUrl())
}
lis := &v3listenerpb.Listener{}
if err := proto.Unmarshal(r.GetValue(), lis); err != nil {
return "", ListenerUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
}
if lis.GetName() == "" {
return "", ListenerUpdate{}, fmt.Errorf("empty resource name in listener resource")
}
lu, err := processListener(lis, bc, sc)
if err != nil {
return lis.GetName(), ListenerUpdate{}, err
}
lu.Raw = r
return lis.GetName(), *lu, nil
}
func processListener(lis *v3listenerpb.Listener, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (*ListenerUpdate, error) {
if lis.GetApiListener() != nil {
return processClientSideListener(lis, bc, sc)
}
return processServerSideListener(lis, bc, sc)
}
// processClientSideListener checks if the provided Listener proto meets
// the expected criteria. If so, it returns a non-empty routeConfigName.
func processClientSideListener(lis *v3listenerpb.Listener, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (*ListenerUpdate, error) {
apiLisAny := lis.GetApiListener().GetApiListener()
if !IsHTTPConnManagerResource(apiLisAny.GetTypeUrl()) {
return nil, fmt.Errorf("unexpected http connection manager resource type: %q", apiLisAny.GetTypeUrl())
}
apiLis := &v3httppb.HttpConnectionManager{}
if err := proto.Unmarshal(apiLisAny.GetValue(), apiLis); err != nil {
return nil, fmt.Errorf("failed to unmarshal api_listener: %v", err)
}
// "HttpConnectionManager.xff_num_trusted_hops must be unset or zero and
// HttpConnectionManager.original_ip_detection_extensions must be empty. If
// either field has an incorrect value, the Listener must be NACKed." - A41
if apiLis.XffNumTrustedHops != 0 {
return nil, fmt.Errorf("xff_num_trusted_hops must be unset or zero %+v", apiLis)
}
if len(apiLis.OriginalIpDetectionExtensions) != 0 {
return nil, fmt.Errorf("original_ip_detection_extensions must be empty %+v", apiLis)
}
hcm := &HTTPConnectionManagerConfig{}
switch apiLis.RouteSpecifier.(type) {
case *v3httppb.HttpConnectionManager_Rds:
if configsource := apiLis.GetRds().GetConfigSource(); configsource.GetAds() == nil && configsource.GetSelf() == nil {
return nil, fmt.Errorf("LDS's RDS configSource is not ADS or Self: %+v", lis)
}
name := apiLis.GetRds().GetRouteConfigName()
if name == "" {
return nil, fmt.Errorf("empty route_config_name: %+v", lis)
}
hcm.RouteConfigName = name
case *v3httppb.HttpConnectionManager_RouteConfig:
routeU, err := generateRDSUpdateFromRouteConfiguration(apiLis.GetRouteConfig(), bc, sc)
if err != nil {
return nil, fmt.Errorf("failed to parse inline RDS resp: %v", err)
}
hcm.InlineRouteConfig = &routeU
case nil:
return nil, fmt.Errorf("no RouteSpecifier: %+v", apiLis)
default:
return nil, fmt.Errorf("unsupported type %T for RouteSpecifier", apiLis.RouteSpecifier)
}
// The following checks and fields only apply to xDS protocol versions v3+.
hcm.MaxStreamDuration = apiLis.GetCommonHttpProtocolOptions().GetMaxStreamDuration().AsDuration()
var err error
if hcm.HTTPFilters, err = processHTTPFilters(apiLis.GetHttpFilters(), false, bc, sc); err != nil {
return nil, err
}
return &ListenerUpdate{APIListener: hcm}, nil
}
func unwrapHTTPFilterConfig(config *anypb.Any) (proto.Message, string, error) {
switch {
case config.MessageIs(&v3xdsxdstypepb.TypedStruct{}):
// The real type name is inside the new TypedStruct message.
s := new(v3xdsxdstypepb.TypedStruct)
if err := config.UnmarshalTo(s); err != nil {
return nil, "", fmt.Errorf("error unmarshalling TypedStruct filter config: %v", err)
}
return s, s.GetTypeUrl(), nil
case config.MessageIs(&v1xdsudpatypepb.TypedStruct{}):
// The real type name is inside the old TypedStruct message.
s := new(v1xdsudpatypepb.TypedStruct)
if err := config.UnmarshalTo(s); err != nil {
return nil, "", fmt.Errorf("error unmarshalling TypedStruct filter config: %v", err)
}
return s, s.GetTypeUrl(), nil
default:
return config, config.GetTypeUrl(), nil
}
}
func validateHTTPFilterConfig(cfg *anypb.Any, lds, optional bool, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (httpfilter.Builder, httpfilter.FilterConfig, error) {
config, typeURL, err := unwrapHTTPFilterConfig(cfg)
if err != nil {
return nil, nil, err
}
filterBuilder := httpfilter.Get(typeURL)
if filterBuilder == nil {
if optional {
return nil, nil, nil
}
return nil, nil, fmt.Errorf("no filter implementation found for %q", typeURL)
}
parseFunc := filterBuilder.ParseFilterConfig
if !lds {
parseFunc = filterBuilder.ParseFilterConfigOverride
}
filterConfig, err := parseFunc(config, httpfilter.ParseOptions{BootstrapConfig: bc, ServerConfig: sc})
if err != nil {
return nil, nil, fmt.Errorf("error parsing config for filter %q: %v", typeURL, err)
}
return filterBuilder, filterConfig, nil
}
func processHTTPFilterOverrides(cfgs map[string]*anypb.Any, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (map[string]httpfilter.FilterConfig, error) {
if len(cfgs) == 0 {
return nil, nil
}
m := make(map[string]httpfilter.FilterConfig)
for name, cfg := range cfgs {
optional := false
disabled := false
s := new(v3routepb.FilterConfig)
if cfg.MessageIs(s) {
if err := cfg.UnmarshalTo(s); err != nil {
return nil, fmt.Errorf("filter override %q: error unmarshalling FilterConfig: %v", name, err)
}
cfg = s.GetConfig()
optional = s.GetIsOptional()
if envconfig.XDSClientExtProcEnabled || envconfig.XDSClientExtAuthzEnabled {
disabled = s.GetDisabled()
}
}
if disabled {
m[name] = httpfilter.DisabledFilterConfig{}
continue
}
httpFilter, config, err := validateHTTPFilterConfig(cfg, false, optional, bc, sc)
if err != nil {
return nil, fmt.Errorf("filter override %q: %v", name, err)
}
if httpFilter == nil {
// Optional configs are ignored.
continue
}
m[name] = config
}
return m, nil
}
func processHTTPFilters(filters []*v3httppb.HttpFilter, server bool, bc *bootstrap.Config, sc *bootstrap.ServerConfig) ([]HTTPFilter, error) {
ret := make([]HTTPFilter, 0, len(filters))
seenNames := make(map[string]bool, len(filters))
for _, filter := range filters {
name := filter.GetName()
if name == "" {
return nil, errors.New("filter missing name field")
}
if seenNames[name] {
return nil, fmt.Errorf("duplicate filter name %q", name)
}
seenNames[name] = true
httpFilter, config, err := validateHTTPFilterConfig(filter.GetTypedConfig(), true, filter.GetIsOptional(), bc, sc)
if err != nil {
return nil, err
}
if httpFilter == nil {
// Optional configs are ignored.
continue
}
if server {
if _, ok := httpFilter.(httpfilter.ServerFilterBuilder); !ok {
if filter.GetIsOptional() {
continue
}
return nil, fmt.Errorf("HTTP filter %q not supported server-side", name)
}
} else if _, ok := httpFilter.(httpfilter.ClientFilterBuilder); !ok {
if filter.GetIsOptional() {
continue
}
return nil, fmt.Errorf("HTTP filter %q not supported client-side", name)
}
disabled := false
if envconfig.XDSClientExtProcEnabled || envconfig.XDSClientExtAuthzEnabled {
disabled = filter.GetDisabled()
}
// Save name/config
ret = append(ret, HTTPFilter{Name: name, Filter: httpFilter, Config: config, Disabled: disabled})
}
// "Validation will fail if a terminal filter is not the last filter in the
// chain or if a non-terminal filter is the last filter in the chain." - A39
if len(ret) == 0 {
return nil, fmt.Errorf("http filters list is empty")
}
var i int
for ; i < len(ret)-1; i++ {
if ret[i].Filter.IsTerminal() {
return nil, fmt.Errorf("http filter %q is a terminal filter but it is not last in the filter chain", ret[i].Name)
}
}
if !ret[i].Filter.IsTerminal() {
return nil, fmt.Errorf("http filter %q is not a terminal filter", ret[len(ret)-1].Name)
}
return ret, nil
}
func processServerSideListener(lis *v3listenerpb.Listener, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (*ListenerUpdate, error) {
if n := len(lis.ListenerFilters); n != 0 {
return nil, fmt.Errorf("unsupported field 'listener_filters' contains %d entries", n)
}
if lis.GetUseOriginalDst().GetValue() {
return nil, errors.New("unsupported field 'use_original_dst' is present and set to true")
}
addr := lis.GetAddress()
if addr == nil {
return nil, fmt.Errorf("no address field in LDS response: %+v", lis)
}
sockAddr := addr.GetSocketAddress()
if sockAddr == nil {
return nil, fmt.Errorf("no socket_address field in LDS response: %+v", lis)
}
lu := &ListenerUpdate{
TCPListener: &InboundListenerConfig{
Address: sockAddr.GetAddress(),
Port: strconv.Itoa(int(sockAddr.GetPortValue())),
},
}
// Populate the default filter chain.
if dfc := lis.GetDefaultFilterChain(); dfc != nil {
fc, err := filterChainFromProto(dfc, bc, sc)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal default filter chain: %v", err)
}
lu.TCPListener.DefaultFilterChain = fc
}
// Populated the filter chain map.
fcm, err := buildFilterChainMap(lis.GetFilterChains(), bc, sc)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal filter chains: %v", err)
}
lu.TCPListener.FilterChains = fcm
// If there are no supported filter chains and no default filter chain, we
// fail here. This will cause the Listener resource to be NACK'ed.
if len(lu.TCPListener.FilterChains.DstPrefixes) == 0 && lu.TCPListener.DefaultFilterChain.IsEmpty() {
return nil, fmt.Errorf("no supported filter chains and no default filter chain")
}
return lu, nil
}
func filterChainFromProto(fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (NetworkFilterChainConfig, error) {
var emptyFilterChain NetworkFilterChainConfig
hcmConfig, err := processNetworkFilters(fc.GetFilters(), bc, sc)
if err != nil {
return emptyFilterChain, err
}
fcc := NetworkFilterChainConfig{HTTPConnMgr: hcmConfig}
// If the transport_socket field is not specified, it means that the control
// plane has not sent us any security config. This is fine and the server
// will use the fallback credentials configured as part of the
// xdsCredentials.
ts := fc.GetTransportSocket()
if ts == nil {
return fcc, nil
}
if name := ts.GetName(); name != transportSocketName {
return emptyFilterChain, fmt.Errorf("transport_socket field has unexpected name: %s", name)
}
tc := ts.GetTypedConfig()
if typeURL := tc.GetTypeUrl(); typeURL != version.V3DownstreamTLSContextURL {
return emptyFilterChain, fmt.Errorf("transport_socket missing typed_config or wrong type_url: %q", typeURL)
}
downstreamCtx := &v3tlspb.DownstreamTlsContext{}
if err := proto.Unmarshal(tc.GetValue(), downstreamCtx); err != nil {
return emptyFilterChain, fmt.Errorf("failed to unmarshal DownstreamTlsContext in LDS response: %v", err)
}
if downstreamCtx.GetRequireSni().GetValue() {
return emptyFilterChain, fmt.Errorf("require_sni field set to true in DownstreamTlsContext message: %v", downstreamCtx)
}
if downstreamCtx.GetOcspStaplePolicy() != v3tlspb.DownstreamTlsContext_LENIENT_STAPLING {
return emptyFilterChain, fmt.Errorf("ocsp_staple_policy field set to unsupported value in DownstreamTlsContext message: %v", downstreamCtx)
}
if downstreamCtx.GetCommonTlsContext() == nil {
return emptyFilterChain, errors.New("DownstreamTlsContext in LDS response does not contain a CommonTlsContext")
}
secCfg, err := securityConfigFromCommonTLSContext(downstreamCtx.GetCommonTlsContext(), true)
if err != nil {
return emptyFilterChain, err
}
if secCfg != nil {
secCfg.RequireClientCert = downstreamCtx.GetRequireClientCertificate().GetValue()
if secCfg.RequireClientCert && secCfg.RootInstanceName == "" {
return emptyFilterChain, errors.New("security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set")
}
fcc.SecurityCfg = secCfg
}
return fcc, nil
}
// dstPrefixEntry wraps DestinationPrefixEntry to track build state.
type dstPrefixEntry struct {
entry DestinationPrefixEntry
rawBufferSeen bool
}
func buildFilterChainMap(fcs []*v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (NetworkFilterChainMap, error) {
dstPrefixEntries := []*dstPrefixEntry{}
for _, fc := range fcs {
fcMatch := fc.GetFilterChainMatch()
if fcMatch.GetDestinationPort().GetValue() != 0 {
// Destination port is the first match criteria and we do not
// support filter chains that contain this match criteria.
logger.Warningf("Dropping filter chain %q since it contains unsupported destination_port match field", fc.GetName())
continue
}
var err error
dstPrefixEntries, err = addFilterChainsForDestPrefixes(dstPrefixEntries, fc, bc, sc)
if err != nil {
return NetworkFilterChainMap{}, err
}
}
entries := []DestinationPrefixEntry{}
for _, bEntry := range dstPrefixEntries {
fcSeen := false
for _, srcPrefixes := range bEntry.entry.SourceTypeArr {
if len(srcPrefixes.Entries) == 0 {
continue
}
for _, srcPrefix := range srcPrefixes.Entries {
for _, fc := range srcPrefix.PortMap {
if !fc.IsEmpty() {
fcSeen = true
}
}
}
}
if fcSeen {
entries = append(entries, bEntry.entry)
}
}
return NetworkFilterChainMap{DstPrefixes: entries}, nil
}
func addFilterChainsForDestPrefixes(dstPrefixEntries []*dstPrefixEntry, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) ([]*dstPrefixEntry, error) {
ranges := fc.GetFilterChainMatch().GetPrefixRanges()
dstPrefixes, err := parsePrefixRanges(ranges)
if err != nil {
return nil, fmt.Errorf("failed to parse destination prefix ranges: %v", err)
}
var entry *dstPrefixEntry
if len(dstPrefixes) == 0 {
// Use the unspecified entry when destination prefix is unspecified, and
// set the `prefix` field to nil.
dstPrefixEntries, entry = getOrCreateDestPrefixEntry(dstPrefixEntries, netip.Prefix{})
if err := addFilterChainsForServerNames(entry, fc, bc, sc); err != nil {
return nil, err
}
return dstPrefixEntries, nil
}
for _, prefix := range dstPrefixes {
dstPrefixEntries, entry = getOrCreateDestPrefixEntry(dstPrefixEntries, prefix)
if err := addFilterChainsForServerNames(entry, fc, bc, sc); err != nil {
return nil, err
}
}
return dstPrefixEntries, nil
}
// getOrCreateDestPrefixEntry looks for an existing dstPrefixEntry in the
// provided slice with the same destination prefix as the provided prefix. If
// such an entry is found, it is returned. Otherwise, a new entry is created and
// appended to the slice, and the new entry is returned.
func getOrCreateDestPrefixEntry(dstPrefixEntries []*dstPrefixEntry, prefix netip.Prefix) ([]*dstPrefixEntry, *dstPrefixEntry) {
for _, e := range dstPrefixEntries {
if e.entry.Prefix == prefix {
return dstPrefixEntries, e
}
}
entry := &dstPrefixEntry{entry: DestinationPrefixEntry{Prefix: prefix}}
dstPrefixEntries = append(dstPrefixEntries, entry)
return dstPrefixEntries, entry
}
func addFilterChainsForServerNames(dstEntry *dstPrefixEntry, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) error {
// Filter chains specifying server names in their match criteria always fail
// a match at connection time. So, these filter chains can be dropped now.
if len(fc.GetFilterChainMatch().GetServerNames()) != 0 {
logger.Warningf("Dropping filter chain %q since it contains unsupported server_names match field", fc.GetName())
return nil
}
return addFilterChainsForTransportProtocols(dstEntry, fc, bc, sc)
}
func addFilterChainsForTransportProtocols(dstEntry *dstPrefixEntry, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) error {
tp := fc.GetFilterChainMatch().GetTransportProtocol()
switch {
case tp != "" && tp != "raw_buffer":
// Only allow filter chains with transport protocol set to empty string
// or "raw_buffer".
logger.Warningf("Dropping filter chain %q since it contains unsupported value for transport_protocol match field", fc.GetName())
return nil
case tp == "" && dstEntry.rawBufferSeen:
// If we have already seen filter chains with transport protocol set to
// "raw_buffer", we can drop filter chains with transport protocol set
// to empty string, since the former takes precedence.
logger.Warningf("Dropping filter chain %q since it contains empty string for transport_protocol match field, but one with raw_buffer already exists", fc.GetName())
return nil
case tp != "" && !dstEntry.rawBufferSeen:
// This is the first "raw_buffer" that we are seeing. Set the bit and
// reset the source types array which might contain entries for filter
// chains with transport protocol set to empty string.
dstEntry.rawBufferSeen = true
dstEntry.entry.SourceTypeArr = [3]SourcePrefixes{}
}
return addFilterChainsForApplicationProtocols(dstEntry, fc, bc, sc)
}
func addFilterChainsForApplicationProtocols(dstEntry *dstPrefixEntry, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) error {
if len(fc.GetFilterChainMatch().GetApplicationProtocols()) != 0 {
logger.Warningf("Dropping filter chain %q since it contains unsupported application_protocols match field", fc.GetName())
return nil
}
return addFilterChainsForSourceType(&dstEntry.entry, fc, bc, sc)
}
// sourceType specifies the connection source IP match type.
type sourceType int
const (
// sourceTypeAny matches connection attempts from any source.
sourceTypeAny sourceType = iota
// sourceTypeSameOrLoopback matches connection attempts from the same host.
sourceTypeSameOrLoopback
// sourceTypeExternal matches connection attempts from a different host.
sourceTypeExternal
)
func addFilterChainsForSourceType(entry *DestinationPrefixEntry, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) error {
var srcType sourceType
switch st := fc.GetFilterChainMatch().GetSourceType(); st {
case v3listenerpb.FilterChainMatch_ANY:
srcType = sourceTypeAny
case v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK:
srcType = sourceTypeSameOrLoopback
case v3listenerpb.FilterChainMatch_EXTERNAL:
srcType = sourceTypeExternal
default:
return fmt.Errorf("unsupported source type: %v", st)
}
return addFilterChainsForSourcePrefixes(&entry.SourceTypeArr[srcType], fc, bc, sc)
}
func addFilterChainsForSourcePrefixes(srcPrefixes *SourcePrefixes, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) error {
ranges := fc.GetFilterChainMatch().GetSourcePrefixRanges()
prefixes, err := parsePrefixRanges(ranges)
if err != nil {
return fmt.Errorf("failed to parse source prefix ranges: %v", err)
}
if len(prefixes) == 0 {
return getOrCreateSourcePrefixEntry(srcPrefixes, netip.Prefix{}, fc, bc, sc)
}
for _, prefix := range prefixes {
if err := getOrCreateSourcePrefixEntry(srcPrefixes, prefix, fc, bc, sc); err != nil {
return err
}
}
return nil
}
func parsePrefixRanges(ranges []*v3corepb.CidrRange) ([]netip.Prefix, error) {
prefixes := make([]netip.Prefix, 0, len(ranges))
for _, pr := range ranges {
addrStr := pr.GetAddressPrefix()
bits := int(pr.GetPrefixLen().GetValue())
addr, err := netip.ParseAddr(addrStr)
if err != nil {
return nil, fmt.Errorf("invalid address: %q", addrStr)
}
prefix := netip.PrefixFrom(addr.Unmap(), bits).Masked()
if !prefix.IsValid() {
return nil, fmt.Errorf(`length %d is invalid for "%s" (max %d)`, bits, addrStr, addr.BitLen())
}
prefixes = append(prefixes, prefix)
}
return prefixes, nil
}
// getOrCreateSourcePrefixEntry looks for an existing SourcePrefixEntry in the
// provided SourcePrefixes with the same source prefix as the provided prefix. If
// such an entry is found, the provided filter chain is added to the entry and
// nil is returned. Otherwise, a new entry is created and appended to the
// SourcePrefixes, the provided filter chain is added to the new entry, and nil
// is returned. If there are multiple filter chains with overlapping matching
// rules, an error is returned.
func getOrCreateSourcePrefixEntry(srcPrefixes *SourcePrefixes, prefix netip.Prefix, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) error {
for i := range srcPrefixes.Entries {
if srcPrefixes.Entries[i].Prefix == prefix {
return addFilterChainsForSourcePorts(&srcPrefixes.Entries[i], fc, bc, sc)
}
}
// Not found, create a new entry.
srcPrefixes.Entries = append(srcPrefixes.Entries, SourcePrefixEntry{
Prefix: prefix,
PortMap: make(map[int]NetworkFilterChainConfig),
})
return addFilterChainsForSourcePorts(&srcPrefixes.Entries[len(srcPrefixes.Entries)-1], fc, bc, sc)
}
func addFilterChainsForSourcePorts(entry *SourcePrefixEntry, fc *v3listenerpb.FilterChain, bc *bootstrap.Config, sc *bootstrap.ServerConfig) error {
ports := fc.GetFilterChainMatch().GetSourcePorts()
srcPorts := make([]int, 0, len(ports))
for _, port := range ports {
srcPorts = append(srcPorts, int(port))
}
if len(srcPorts) == 0 {
if !entry.PortMap[0].IsEmpty() {
return errors.New("multiple filter chains with overlapping matching rules are defined")
}
fcc, err := filterChainFromProto(fc, bc, sc)
if err != nil {
return err
}
entry.PortMap[0] = fcc
return nil
}
for _, port := range srcPorts {
if !entry.PortMap[port].IsEmpty() {
return errors.New("multiple filter chains with overlapping matching rules are defined")
}
fcc, err := filterChainFromProto(fc, bc, sc)
if err != nil {
return err
}
entry.PortMap[port] = fcc
}
return nil
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,464 @@
/*
*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"fmt"
"math"
"regexp"
"strings"
"time"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/internal/envconfig"
"google.golang.org/grpc/internal/xds/bootstrap"
"google.golang.org/grpc/internal/xds/clusterspecifier"
"google.golang.org/grpc/internal/xds/matcher"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3"
)
func unmarshalRouteConfigResource(r *anypb.Any, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (string, RouteConfigUpdate, error) {
r, err := UnwrapResource(r)
if err != nil {
return "", RouteConfigUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err)
}
if !IsRouteConfigResource(r.GetTypeUrl()) {
return "", RouteConfigUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl())
}
rc := &v3routepb.RouteConfiguration{}
if err := proto.Unmarshal(r.GetValue(), rc); err != nil {
return "", RouteConfigUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
}
if rc.GetName() == "" {
return "", RouteConfigUpdate{}, fmt.Errorf("empty resource name in route config resource")
}
u, err := generateRDSUpdateFromRouteConfiguration(rc, bc, sc)
if err != nil {
return rc.GetName(), RouteConfigUpdate{}, err
}
u.Raw = r
return rc.GetName(), u, nil
}
// generateRDSUpdateFromRouteConfiguration checks if the provided
// RouteConfiguration meets the expected criteria. If so, it returns a
// RouteConfigUpdate with nil error.
//
// A RouteConfiguration resource is considered valid when only if it contains a
// VirtualHost whose domain field matches the server name from the URI passed
// to the gRPC channel, and it contains a clusterName or a weighted cluster.
//
// The RouteConfiguration includes a list of virtualHosts, which may have zero
// or more elements. We are interested in the element whose domains field
// matches the server name specified in the "xds:" URI. The only field in the
// VirtualHost proto that we are interested in is the list of routes. We
// only look at the last route in the list (the default route), whose match
// field must be empty and whose route field must be set. Inside that route
// message, the cluster field will contain the clusterName or weighted clusters
// we are looking for.
func generateRDSUpdateFromRouteConfiguration(rc *v3routepb.RouteConfiguration, bc *bootstrap.Config, sc *bootstrap.ServerConfig) (RouteConfigUpdate, error) {
vhs := make([]*VirtualHost, 0, len(rc.GetVirtualHosts()))
csps, err := processClusterSpecifierPlugins(rc.ClusterSpecifierPlugins)
if err != nil {
return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err)
}
// cspNames represents all the cluster specifiers referenced by Route
// Actions - any cluster specifiers not referenced by a Route Action can be
// ignored and not emitted by the xdsclient.
var cspNames = make(map[string]bool)
for _, vh := range rc.GetVirtualHosts() {
routes, cspNs, err := routesProtoToSlice(vh.Routes, csps, bc, sc)
if err != nil {
return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err)
}
for n := range cspNs {
cspNames[n] = true
}
rc, err := generateRetryConfig(vh.GetRetryPolicy())
if err != nil {
return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err)
}
vhOut := &VirtualHost{
Domains: vh.GetDomains(),
Routes: routes,
RetryConfig: rc,
}
cfgs, err := processHTTPFilterOverrides(vh.GetTypedPerFilterConfig(), bc, sc)
if err != nil {
return RouteConfigUpdate{}, fmt.Errorf("virtual host %+v: %v", vh, err)
}
vhOut.HTTPFilterConfigOverride = cfgs
vhs = append(vhs, vhOut)
}
// "For any entry in the RouteConfiguration.cluster_specifier_plugins not
// referenced by an enclosed ActionType's cluster_specifier_plugin, the xDS
// client should not provide it to its consumers." - RLS in xDS Design
for name := range csps {
if !cspNames[name] {
delete(csps, name)
}
}
return RouteConfigUpdate{VirtualHosts: vhs, ClusterSpecifierPlugins: csps}, nil
}
func processClusterSpecifierPlugins(csps []*v3routepb.ClusterSpecifierPlugin) (map[string]clusterspecifier.BalancerConfig, error) {
cspCfgs := make(map[string]clusterspecifier.BalancerConfig)
// "The xDS client will inspect all elements of the
// cluster_specifier_plugins field looking up a plugin based on the
// extension.typed_config of each." - RLS in xDS design
for _, csp := range csps {
cs := clusterspecifier.Get(csp.GetExtension().GetTypedConfig().GetTypeUrl())
if cs == nil {
if csp.GetIsOptional() {
// "If a plugin is not supported but has is_optional set, then
// we will ignore any routes that point to that plugin"
cspCfgs[csp.GetExtension().GetName()] = nil
continue
}
// "If no plugin is registered for it, the resource will be NACKed."
// - RLS in xDS design
return nil, fmt.Errorf("cluster specifier %q of type %q was not found", csp.GetExtension().GetName(), csp.GetExtension().GetTypedConfig().GetTypeUrl())
}
lbCfg, err := cs.ParseClusterSpecifierConfig(csp.GetExtension().GetTypedConfig())
if err != nil {
// "If a plugin is found, the value of the typed_config field will
// be passed to it's conversion method, and if an error is
// encountered, the resource will be NACKED." - RLS in xDS design
return nil, fmt.Errorf("error: %q parsing config %q for cluster specifier %q of type %q", err, csp.GetExtension().GetTypedConfig(), csp.GetExtension().GetName(), csp.GetExtension().GetTypedConfig().GetTypeUrl())
}
// "If all cluster specifiers are valid, the xDS client will store the
// configurations in a map keyed by the name of the extension instance." -
// RLS in xDS Design
cspCfgs[csp.GetExtension().GetName()] = lbCfg
}
return cspCfgs, nil
}
func generateRetryConfig(rp *v3routepb.RetryPolicy) (*RetryConfig, error) {
if rp == nil {
return nil, nil
}
cfg := &RetryConfig{RetryOn: make(map[codes.Code]bool)}
for _, s := range strings.Split(rp.GetRetryOn(), ",") {
switch strings.TrimSpace(strings.ToLower(s)) {
case "cancelled":
cfg.RetryOn[codes.Canceled] = true
case "deadline-exceeded":
cfg.RetryOn[codes.DeadlineExceeded] = true
case "internal":
cfg.RetryOn[codes.Internal] = true
case "resource-exhausted":
cfg.RetryOn[codes.ResourceExhausted] = true
case "unavailable":
cfg.RetryOn[codes.Unavailable] = true
}
}
if rp.NumRetries == nil {
cfg.NumRetries = 1
} else {
cfg.NumRetries = rp.GetNumRetries().Value
if cfg.NumRetries < 1 {
return nil, fmt.Errorf("retry_policy.num_retries = %v; must be >= 1", cfg.NumRetries)
}
}
backoff := rp.GetRetryBackOff()
if backoff == nil {
cfg.RetryBackoff.BaseInterval = 25 * time.Millisecond
} else {
cfg.RetryBackoff.BaseInterval = backoff.GetBaseInterval().AsDuration()
if cfg.RetryBackoff.BaseInterval <= 0 {
return nil, fmt.Errorf("retry_policy.base_interval = %v; must be > 0", cfg.RetryBackoff.BaseInterval)
}
}
if max := backoff.GetMaxInterval(); max == nil {
cfg.RetryBackoff.MaxInterval = 10 * cfg.RetryBackoff.BaseInterval
} else {
cfg.RetryBackoff.MaxInterval = max.AsDuration()
if cfg.RetryBackoff.MaxInterval <= 0 {
return nil, fmt.Errorf("retry_policy.max_interval = %v; must be > 0", cfg.RetryBackoff.MaxInterval)
}
}
if len(cfg.RetryOn) == 0 {
return &RetryConfig{}, nil
}
return cfg, nil
}
func routesProtoToSlice(routes []*v3routepb.Route, csps map[string]clusterspecifier.BalancerConfig, bc *bootstrap.Config, sc *bootstrap.ServerConfig) ([]*Route, map[string]bool, error) {
var routesRet []*Route
var cspNames = make(map[string]bool)
for _, r := range routes {
match := r.GetMatch()
if match == nil {
return nil, nil, fmt.Errorf("route %+v doesn't have a match", r)
}
if len(match.GetQueryParameters()) != 0 {
// Ignore route with query parameters.
logger.Warningf("Ignoring route %+v with query parameter matchers", r)
continue
}
pathSp := match.GetPathSpecifier()
if pathSp == nil {
return nil, nil, fmt.Errorf("route %+v doesn't have a path specifier", r)
}
var route Route
switch pt := pathSp.(type) {
case *v3routepb.RouteMatch_Prefix:
route.Prefix = &pt.Prefix
case *v3routepb.RouteMatch_Path:
route.Path = &pt.Path
case *v3routepb.RouteMatch_SafeRegex:
regex := pt.SafeRegex.GetRegex()
re, err := matcher.CompileSafeRegex(regex)
if err != nil {
return nil, nil, fmt.Errorf("route %+v contains an invalid regex %q", r, regex)
}
route.Regex = re
default:
return nil, nil, fmt.Errorf("route %+v has an unrecognized path specifier: %+v", r, pt)
}
if caseSensitive := match.GetCaseSensitive(); caseSensitive != nil {
route.CaseInsensitive = !caseSensitive.Value
}
for _, h := range match.GetHeaders() {
var header HeaderMatcher
// The deprecated exact/prefix/suffix/contains match fields are
// converted to the equivalent StringMatcher so that
// StringMatcherFromProto owns validation for all of them.
var smProto *v3matcherpb.StringMatcher
switch ht := h.GetHeaderMatchSpecifier().(type) {
case *v3routepb.HeaderMatcher_ExactMatch:
smProto = &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ht.ExactMatch}}
case *v3routepb.HeaderMatcher_SafeRegexMatch:
regex := ht.SafeRegexMatch.GetRegex()
re, err := matcher.CompileSafeRegex(regex)
if err != nil {
return nil, nil, fmt.Errorf("route %+v contains an invalid regex %q", r, regex)
}
header.RegexMatch = re
case *v3routepb.HeaderMatcher_RangeMatch:
header.RangeMatch = &Int64Range{
Start: ht.RangeMatch.Start,
End: ht.RangeMatch.End,
}
case *v3routepb.HeaderMatcher_PresentMatch:
header.PresentMatch = &ht.PresentMatch
case *v3routepb.HeaderMatcher_PrefixMatch:
smProto = &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: ht.PrefixMatch}}
case *v3routepb.HeaderMatcher_SuffixMatch:
smProto = &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: ht.SuffixMatch}}
case *v3routepb.HeaderMatcher_ContainsMatch:
smProto = &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: ht.ContainsMatch}}
case *v3routepb.HeaderMatcher_StringMatch:
if ht.StringMatch == nil {
return nil, nil, fmt.Errorf("route %+v has an empty string matcher", r)
}
smProto = ht.StringMatch
default:
return nil, nil, fmt.Errorf("route %+v has an unrecognized header matcher: %+v", r, ht)
}
if smProto != nil {
sm, err := matcher.StringMatcherFromProto(smProto)
if err != nil {
return nil, nil, fmt.Errorf("route %+v has an invalid header matcher: %v", r, err)
}
header.StringMatch = &sm
}
// The metadata the matchers run against always has lowercase keys,
// so a name that contains an uppercase character matches no header
// and the route holding it never fires.
header.Name = strings.ToLower(h.GetName())
invert := h.GetInvertMatch()
header.InvertMatch = &invert
route.Headers = append(route.Headers, &header)
}
if fr := match.GetRuntimeFraction(); fr != nil {
d := fr.GetDefaultValue()
n := d.GetNumerator()
switch d.GetDenominator() {
case v3typepb.FractionalPercent_HUNDRED:
n *= 10000
case v3typepb.FractionalPercent_TEN_THOUSAND:
n *= 100
case v3typepb.FractionalPercent_MILLION:
}
route.Fraction = &n
}
switch r.GetAction().(type) {
case *v3routepb.Route_Route:
action := r.GetRoute()
if envconfig.XDSAuthorityRewrite {
if sc != nil && sc.ServerFeaturesTrustedXDSServer() {
route.AutoHostRewrite = action.GetAutoHostRewrite().GetValue()
}
}
// Hash Policies are only applicable for a Ring Hash LB.
hp, err := hashPoliciesProtoToSlice(action.HashPolicy)
if err != nil {
return nil, nil, err
}
route.HashPolicies = hp
switch a := action.GetClusterSpecifier().(type) {
case *v3routepb.RouteAction_Cluster:
route.WeightedClusters = append(route.WeightedClusters, WeightedCluster{Name: a.Cluster, Weight: 1})
case *v3routepb.RouteAction_WeightedClusters:
wcs := a.WeightedClusters
var totalWeight uint64
for _, c := range wcs.Clusters {
w := c.GetWeight().GetValue()
if w == 0 {
continue
}
totalWeight += uint64(w)
if totalWeight > math.MaxUint32 {
return nil, nil, fmt.Errorf("xds: total weight of clusters exceeds MaxUint32")
}
wc := WeightedCluster{Name: c.GetName(), Weight: w}
cfgs, err := processHTTPFilterOverrides(c.GetTypedPerFilterConfig(), bc, sc)
if err != nil {
return nil, nil, fmt.Errorf("route %+v, action %+v: %v", r, a, err)
}
wc.HTTPFilterConfigOverride = cfgs
route.WeightedClusters = append(route.WeightedClusters, wc)
}
if totalWeight == 0 {
return nil, nil, fmt.Errorf("route %+v, action %+v, has no valid cluster in WeightedCluster action", r, a)
}
case *v3routepb.RouteAction_ClusterSpecifierPlugin:
// gRFC A28 was updated to say the following:
//
// The route’s action field must be route, and its
// cluster_specifier:
// - Can be Cluster
// - Can be Weighted_clusters
// - Can be unset or an unsupported field. The route containing
// this action will be ignored.
//
// This means that if this env var is not set, we should treat
// it as if it we didn't know about the cluster_specifier_plugin
// at all.
if _, ok := csps[a.ClusterSpecifierPlugin]; !ok {
// "When processing RouteActions, if any action includes a
// cluster_specifier_plugin value that is not in
// RouteConfiguration.cluster_specifier_plugins, the
// resource will be NACKed." - RLS in xDS design
return nil, nil, fmt.Errorf("route %+v, action %+v, specifies a cluster specifier plugin %+v that is not in Route Configuration", r, a, a.ClusterSpecifierPlugin)
}
if csps[a.ClusterSpecifierPlugin] == nil {
logger.Warningf("Ignoring route %+v with optional and unsupported cluster specifier plugin %+v", r, a.ClusterSpecifierPlugin)
continue
}
cspNames[a.ClusterSpecifierPlugin] = true
route.ClusterSpecifierPlugin = a.ClusterSpecifierPlugin
default:
logger.Warningf("Ignoring route %+v with unknown ClusterSpecifier %+v", r, a)
continue
}
msd := action.GetMaxStreamDuration()
// Prefer grpc_timeout_header_max, if set.
dur := msd.GetGrpcTimeoutHeaderMax()
if dur == nil {
dur = msd.GetMaxStreamDuration()
}
if dur != nil {
d := dur.AsDuration()
route.MaxStreamDuration = &d
}
route.RetryConfig, err = generateRetryConfig(action.GetRetryPolicy())
if err != nil {
return nil, nil, fmt.Errorf("route %+v, action %+v: %v", r, action, err)
}
route.ActionType = RouteActionRoute
case *v3routepb.Route_NonForwardingAction:
// Expected to be used on server side.
route.ActionType = RouteActionNonForwardingAction
default:
route.ActionType = RouteActionUnsupported
}
cfgs, err := processHTTPFilterOverrides(r.GetTypedPerFilterConfig(), bc, sc)
if err != nil {
return nil, nil, fmt.Errorf("route %+v: %v", r, err)
}
route.HTTPFilterConfigOverride = cfgs
routesRet = append(routesRet, &route)
}
return routesRet, cspNames, nil
}
func hashPoliciesProtoToSlice(policies []*v3routepb.RouteAction_HashPolicy) ([]*HashPolicy, error) {
var hashPoliciesRet []*HashPolicy
for _, p := range policies {
policy := HashPolicy{Terminal: p.Terminal}
switch p.GetPolicySpecifier().(type) {
case *v3routepb.RouteAction_HashPolicy_Header_:
policy.HashPolicyType = HashPolicyTypeHeader
policy.HeaderName = p.GetHeader().GetHeaderName()
if rr := p.GetHeader().GetRegexRewrite(); rr != nil {
regex := rr.GetPattern().GetRegex()
re, err := regexp.Compile(regex)
if err != nil {
return nil, fmt.Errorf("hash policy %+v contains an invalid regex %q", p, regex)
}
policy.Regex = re
policy.RegexSubstitution = rr.GetSubstitution()
}
case *v3routepb.RouteAction_HashPolicy_FilterState_:
if p.GetFilterState().GetKey() != "io.grpc.channel_id" {
logger.Warningf("Ignoring hash policy %+v with invalid key for filter state policy %q", p, p.GetFilterState().GetKey())
continue
}
policy.HashPolicyType = HashPolicyTypeChannelID
default:
logger.Warningf("Ignoring unsupported hash policy %T", p.GetPolicySpecifier())
continue
}
hashPoliciesRet = append(hashPoliciesRet, &policy)
}
return hashPoliciesRet, nil
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,44 @@
/*
*
* Copyright 2020 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
// Package version defines constants to distinguish between supported xDS API
// versions.
package version
// Resource URLs. We need to be able to accept either version of the resource
// regardless of the version of the transport protocol in use.
const (
googleapiPrefix = "type.googleapis.com/"
V3ListenerType = "envoy.config.listener.v3.Listener"
V3RouteConfigType = "envoy.config.route.v3.RouteConfiguration"
V3ClusterType = "envoy.config.cluster.v3.Cluster"
V3EndpointsType = "envoy.config.endpoint.v3.ClusterLoadAssignment"
V3ResourceWrapperURL = googleapiPrefix + "envoy.service.discovery.v3.Resource"
V3ListenerURL = googleapiPrefix + V3ListenerType
V3RouteConfigURL = googleapiPrefix + V3RouteConfigType
V3ClusterURL = googleapiPrefix + V3ClusterType
V3EndpointsURL = googleapiPrefix + V3EndpointsType
V3HTTPConnManagerURL = googleapiPrefix + "envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager"
V3UpstreamTLSContextURL = googleapiPrefix + "envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext"
V3DownstreamTLSContextURL = googleapiPrefix + "envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext"
V3HTTP11ProxyUpstreamTransportURL = googleapiPrefix + "envoy.extensions.transport_sockets.http_11_proxy.v3.Http11ProxyUpstreamTransport"
V3AddressURL = googleapiPrefix + "envoy.config.core.v3.Address"
V3AudienceURL = googleapiPrefix + "envoy.extensions.filters.http.gcp_authn.v3.Audience"
)

View File

@@ -0,0 +1,133 @@
/*
*
* Copyright 2025 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package xdsresource
import (
"context"
"google.golang.org/grpc/resolver"
)
// XDSConfig holds the complete gRPC client-side xDS configuration containing
// all necessary resources.
type XDSConfig struct {
// Listener holds the listener configuration. It is guaranteed to be
// non-nil.
Listener *ListenerUpdate
// RouteConfig holds the route configuration. It will be populated even if
// the route configuration was inlined into the Listener resource. It is
// guaranteed to be non-nil.
RouteConfig *RouteConfigUpdate
// VirtualHost is selected from the route configuration whose domain field
// offers the best match against the provided dataplane authority. It is
// guaranteed to be non-nil.
VirtualHost *VirtualHost
// Clusters is a map from cluster name to its configuration.
Clusters map[string]*ClusterResult
}
// ClusterResult contains a cluster's configuration when a valid resource is
// received from the management server. It contains an error when:
// - an invalid resource is received from the management server and
// a valid resource was not already present or
// - the cluster resource does not exist on the management server
type ClusterResult struct {
Config ClusterConfig
Err error
}
// ClusterConfig contains configuration for a single cluster.
type ClusterConfig struct {
// Cluster configuration for the cluster. This field is always set to a
// non-nil value.
Cluster *ClusterUpdate
// EndpointConfig contains endpoint configuration for a leaf cluster. This
// field is only set for EDS and LOGICAL_DNS clusters.
EndpointConfig *EndpointConfig
// AggregateConfig contains configuration for an aggregate cluster. This
// field is only set for AGGREGATE clusters.
AggregateConfig *AggregateConfig
}
// AggregateConfig holds the configuration for an aggregate cluster.
type AggregateConfig struct {
// LeafClusters contains a prioritized list of names of the leaf clusters
// for the cluster.
LeafClusters []string
}
// EndpointConfig contains configuration corresponding to the endpoints in a
// cluster. Only one of EDSUpdate or DNSEndpoints will be populated based on the
// cluster type.
type EndpointConfig struct {
// Endpoint configurartion for the EDS clusters.
EDSUpdate *EndpointsUpdate
// Endpoint configuration for the LOGICAL_DNS clusters.
DNSEndpoints *DNSUpdate
// ResolutionNote stores error encountered while obtaining endpoints data
// for the cluster. It will contain a nil value when a valid endpoint data is
// received. It contains an error when:
// - an invalid resource is received from the management server or
// - the endpoint resource does not exist on the management server
ResolutionNote error
}
// DNSUpdate represents the result of a DNS resolution, containing a list of
// discovered endpoints.
type DNSUpdate struct {
// Endpoints is the complete list of endpoints returned by the DNS resolver.
Endpoints []resolver.Endpoint
}
// xdsConfigkey is the type used as the key to store XDSConfig in the Attributes
// field of resolver.State.
type xdsConfigkey struct{}
// SetXDSConfig returns a copy of state in which the Attributes field is updated
// with the XDSConfig.
func SetXDSConfig(state resolver.State, config *XDSConfig) resolver.State {
state.Attributes = state.Attributes.WithValue(xdsConfigkey{}, config)
return state
}
// XDSConfigFromResolverState returns XDSConfig stored as an attribute in the
// resolver state.
func XDSConfigFromResolverState(state resolver.State) *XDSConfig {
if v := state.Attributes.Value(xdsConfigkey{}); v != nil {
return v.(*XDSConfig)
}
return nil
}
type xdsConfigKey struct{}
// NewContextWithXDSConfig returns a new context with xDSConfig added to it.
func NewContextWithXDSConfig(ctx context.Context, config *XDSConfig) context.Context {
return context.WithValue(ctx, xdsConfigKey{}, config)
}
// XDSConfigFromContext returns the XDSConfig stored in ctx.
func XDSConfigFromContext(ctx context.Context) *XDSConfig {
if v := ctx.Value(xdsConfigKey{}); v != nil {
return v.(*XDSConfig)
}
return nil
}