snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)
Some checks failed
codecov / upload (push) Failing after 13m59s
CodeQL / Analyze (push) Failing after 1s
govulncheck / govulncheck (.) (push) Failing after 1s
govulncheck / govulncheck (./cmd/protoc-gen-go-grpc) (push) Failing after 1s
govulncheck / govulncheck (./gcp/observability) (push) Failing after 1s
govulncheck / govulncheck (./security/advancedtls) (push) Failing after 1s
govulncheck / govulncheck (./stats/opencensus) (push) Failing after 1s
Testing / tests (arm, latest) (push) Waiting to run
Testing / vet-proto (push) Failing after 1s
Testing / extras (latest) (push) Failing after 1s
Testing / static checks (latest-1) (push) Failing after 1s
Testing / tests (-race, latest) (push) Failing after 1s
Testing / tests (i386, latest) (push) Failing after 1s
Testing / tests (latest) (push) Failing after 1s
Testing / tests (latest-1) (push) Failing after 1s
Lock Threads / lock (push) Failing after 1s
Stale bot / stale (push) Failing after 1s
Some checks failed
codecov / upload (push) Failing after 13m59s
CodeQL / Analyze (push) Failing after 1s
govulncheck / govulncheck (.) (push) Failing after 1s
govulncheck / govulncheck (./cmd/protoc-gen-go-grpc) (push) Failing after 1s
govulncheck / govulncheck (./gcp/observability) (push) Failing after 1s
govulncheck / govulncheck (./security/advancedtls) (push) Failing after 1s
govulncheck / govulncheck (./stats/opencensus) (push) Failing after 1s
Testing / tests (arm, latest) (push) Waiting to run
Testing / vet-proto (push) Failing after 1s
Testing / extras (latest) (push) Failing after 1s
Testing / static checks (latest-1) (push) Failing after 1s
Testing / tests (-race, latest) (push) Failing after 1s
Testing / tests (i386, latest) (push) Failing after 1s
Testing / tests (latest) (push) Failing after 1s
Testing / tests (latest-1) (push) Failing after 1s
Lock Threads / lock (push) Failing after 1s
Stale bot / stale (push) Failing after 1s
This commit is contained in:
733
internal/xds/clients/xdsclient/ads_stream.go
Normal file
733
internal/xds/clients/xdsclient/ads_stream.go
Normal file
@@ -0,0 +1,733 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc/grpclog"
|
||||
igrpclog "google.golang.org/grpc/internal/grpclog"
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/backoff"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/pretty"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
|
||||
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
|
||||
cpb "google.golang.org/genproto/googleapis/rpc/code"
|
||||
statuspb "google.golang.org/genproto/googleapis/rpc/status"
|
||||
)
|
||||
|
||||
const (
|
||||
// Any per-RPC level logs which print complete request or response messages
|
||||
// should be gated at this verbosity level. Other per-RPC level logs which print
|
||||
// terse output should be at `INFO` and verbosity 2.
|
||||
perRPCVerbosityLevel = 9
|
||||
)
|
||||
|
||||
// request represents a queued request message to be sent on the ADS stream. It
|
||||
// contains the type of the resource and the list of resource names to be sent.
|
||||
type request struct {
|
||||
typ ResourceType
|
||||
resourceNames []string
|
||||
}
|
||||
|
||||
// response represents a response received on the ADS stream. It contains the
|
||||
// type URL, version, and resources for the response.
|
||||
type response struct {
|
||||
typeURL string
|
||||
version string
|
||||
resources []*anypb.Any
|
||||
}
|
||||
|
||||
// dataAndErrTuple is a struct that holds a resource and an error. It is used to
|
||||
// return a resource and any associated error from a function.
|
||||
type dataAndErrTuple struct {
|
||||
Resource ResourceData
|
||||
Err error
|
||||
}
|
||||
|
||||
// adsStreamEventHandler is an interface that defines the callbacks for events that
|
||||
// occur on the ADS stream. Methods on this interface may be invoked
|
||||
// concurrently and implementations need to handle them in a thread-safe manner.
|
||||
type adsStreamEventHandler interface {
|
||||
onStreamError(error) // Called when the ADS stream breaks.
|
||||
onWatchExpiry(ResourceType, string) // Called when the watch timer expires for a resource.
|
||||
onResponse(response, func()) ([]string, error) // Called when a response is received on the ADS stream.
|
||||
}
|
||||
|
||||
// state corresponding to a resource type.
|
||||
type resourceTypeState struct {
|
||||
version string // Last acked version. Should not be reset when the stream breaks.
|
||||
nonce string // Last received nonce. Should be reset when the stream breaks.
|
||||
subscribedResources map[string]*xdsresource.ResourceWatchState // Map of subscribed resource names to their state.
|
||||
}
|
||||
|
||||
// adsStreamImpl provides the functionality associated with an ADS (Aggregated
|
||||
// Discovery Service) stream on the client side. It manages the lifecycle of the
|
||||
// ADS stream, including creating the stream, sending requests, and handling
|
||||
// responses. It also handles flow control and retries for the stream.
|
||||
type adsStreamImpl struct {
|
||||
// The following fields are initialized from arguments passed to the
|
||||
// constructor and are read-only afterwards, and hence can be accessed
|
||||
// without a mutex.
|
||||
transport clients.Transport // Transport to use for ADS stream.
|
||||
eventHandler adsStreamEventHandler // Callbacks into the xdsChannel.
|
||||
backoff func(int) time.Duration // Backoff for retries, after stream failures.
|
||||
nodeProto *v3corepb.Node // Identifies the gRPC application.
|
||||
watchExpiryTimeout time.Duration // Resource watch expiry timeout
|
||||
logger *igrpclog.PrefixLogger
|
||||
|
||||
// The following fields are initialized in the constructor and are not
|
||||
// written to afterwards, and hence can be accessed without a mutex.
|
||||
streamCh chan clients.Stream // New ADS streams are pushed here.
|
||||
runnerDoneCh chan struct{} // Notify completion of runner goroutine.
|
||||
cancel context.CancelFunc // To cancel the context passed to the runner goroutine.
|
||||
fc *adsFlowControl // Flow control for ADS stream.
|
||||
notifySender chan struct{} // To notify the sending goroutine of a pending request.
|
||||
|
||||
// Guards access to the below fields (and to the contents of the map).
|
||||
mu sync.Mutex
|
||||
resourceTypeState map[ResourceType]*resourceTypeState // Map of resource types to their state.
|
||||
pendingRequests []request // Subscriptions and unsubscriptions are pushed here.
|
||||
|
||||
// The following fields are accessed atomically.
|
||||
firstRequest atomic.Bool // False after the first request is sent out.
|
||||
firstStreamCreated atomic.Bool // Set to true after the very first ADS stream is created.
|
||||
streamEstablished atomic.Bool // Set to true when an ADS stream is established and a response is received, except for the very first stream which is set to true immediately.
|
||||
}
|
||||
|
||||
// adsStreamOpts contains the options for creating a new ADS Stream.
|
||||
type adsStreamOpts struct {
|
||||
transport clients.Transport // xDS transport to create the stream on.
|
||||
eventHandler adsStreamEventHandler // Callbacks for stream events.
|
||||
backoff func(int) time.Duration // Backoff for retries, after stream failures.
|
||||
nodeProto *v3corepb.Node // Node proto to identify the gRPC application.
|
||||
watchExpiryTimeout time.Duration // Resource watch expiry timeout.
|
||||
logPrefix string // Prefix to be used for log messages.
|
||||
}
|
||||
|
||||
// newADSStreamImpl initializes a new adsStreamImpl instance using the given
|
||||
// parameters. It also launches goroutines responsible for managing reads and
|
||||
// writes for messages of the underlying stream.
|
||||
func newADSStreamImpl(opts adsStreamOpts) *adsStreamImpl {
|
||||
s := &adsStreamImpl{
|
||||
transport: opts.transport,
|
||||
eventHandler: opts.eventHandler,
|
||||
backoff: opts.backoff,
|
||||
nodeProto: opts.nodeProto,
|
||||
watchExpiryTimeout: opts.watchExpiryTimeout,
|
||||
|
||||
streamCh: make(chan clients.Stream, 1),
|
||||
runnerDoneCh: make(chan struct{}),
|
||||
fc: newADSFlowControl(),
|
||||
notifySender: make(chan struct{}, 1),
|
||||
resourceTypeState: make(map[ResourceType]*resourceTypeState),
|
||||
}
|
||||
|
||||
l := grpclog.Component("xds")
|
||||
s.logger = igrpclog.NewPrefixLogger(l, opts.logPrefix+fmt.Sprintf("[ads-stream %p] ", s))
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s.cancel = cancel
|
||||
go s.runner(ctx)
|
||||
return s
|
||||
}
|
||||
|
||||
// Stop blocks until the stream is closed and all spawned goroutines exit.
|
||||
func (s *adsStreamImpl) Stop() {
|
||||
s.cancel()
|
||||
s.fc.stop()
|
||||
<-s.runnerDoneCh
|
||||
s.logger.Infof("Shutdown ADS stream")
|
||||
}
|
||||
|
||||
// subscribe subscribes to the given resource. It is assumed that multiple
|
||||
// subscriptions for the same resource is deduped at the caller. A discovery
|
||||
// request is sent out on the underlying stream, for the resource type with the
|
||||
// newly subscribed resource.
|
||||
func (s *adsStreamImpl) subscribe(typ ResourceType, name string) {
|
||||
if s.logger.V(2) {
|
||||
s.logger.Infof("Subscribing to resource %q of type %q", name, typ.TypeName)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
state, ok := s.resourceTypeState[typ]
|
||||
if !ok {
|
||||
// An entry in the type state map is created as part of the first
|
||||
// subscription request for this type.
|
||||
state = &resourceTypeState{subscribedResources: make(map[string]*xdsresource.ResourceWatchState)}
|
||||
s.resourceTypeState[typ] = state
|
||||
}
|
||||
|
||||
// Create state for the newly subscribed resource. The watch timer will
|
||||
// be started when a request for this resource is actually sent out.
|
||||
state.subscribedResources[name] = &xdsresource.ResourceWatchState{State: xdsresource.ResourceWatchStateStarted}
|
||||
|
||||
// Send a request for the resource type with updated subscriptions.
|
||||
s.pendingRequests = append(s.pendingRequests, request{typ: typ, resourceNames: resourceNames(state.subscribedResources)})
|
||||
s.mu.Unlock()
|
||||
|
||||
select {
|
||||
case s.notifySender <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// unsubscribe cancels the subscription to the given resource. It is a no-op if
|
||||
// the given resource does not exist. The watch expiry timer associated with the
|
||||
// resource is stopped if one is active. A discovery request is sent out on the
|
||||
// stream for the resource type with the updated set of resource names.
|
||||
func (s *adsStreamImpl) unsubscribe(typ ResourceType, name string) {
|
||||
if s.logger.V(2) {
|
||||
s.logger.Infof("Unsubscribing to resource %q of type %q", name, typ.TypeName)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
state, ok := s.resourceTypeState[typ]
|
||||
if !ok {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
rs, ok := state.subscribedResources[name]
|
||||
if !ok {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if rs.ExpiryTimer != nil {
|
||||
rs.ExpiryTimer.Stop()
|
||||
}
|
||||
delete(state.subscribedResources, name)
|
||||
|
||||
// Send a request for the resource type with updated subscriptions.
|
||||
s.pendingRequests = append(s.pendingRequests, request{typ: typ, resourceNames: resourceNames(state.subscribedResources)})
|
||||
s.mu.Unlock()
|
||||
|
||||
select {
|
||||
case s.notifySender <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// runner is a long-running goroutine that handles the lifecycle of the ADS
|
||||
// stream. It spawns another goroutine to handle writes of discovery request
|
||||
// messages on the stream. Whenever an existing stream fails, it performs
|
||||
// exponential backoff (if no messages were received on that stream) before
|
||||
// creating a new stream.
|
||||
func (s *adsStreamImpl) runner(ctx context.Context) {
|
||||
defer close(s.runnerDoneCh)
|
||||
|
||||
go s.send(ctx)
|
||||
|
||||
runStreamWithBackoff := func() error {
|
||||
stream, err := s.transport.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources")
|
||||
if err != nil {
|
||||
s.logger.Warningf("Failed to create a new ADS streaming RPC: %v", err)
|
||||
s.streamEstablished.Store(false)
|
||||
s.onError(err, false)
|
||||
return nil
|
||||
}
|
||||
if s.logger.V(2) {
|
||||
s.logger.Infof("ADS stream created")
|
||||
}
|
||||
|
||||
s.firstRequest.Store(true)
|
||||
if !s.firstStreamCreated.Load() {
|
||||
s.streamEstablished.Store(true)
|
||||
s.firstStreamCreated.Store(true)
|
||||
}
|
||||
|
||||
// Ensure that the most recently created stream is pushed on the
|
||||
// channel for the `send` goroutine to consume.
|
||||
select {
|
||||
case <-s.streamCh:
|
||||
default:
|
||||
}
|
||||
s.streamCh <- stream
|
||||
|
||||
// Backoff state is reset upon successful receipt of at least one
|
||||
// message from the server.
|
||||
if s.recv(stream) {
|
||||
return backoff.ErrResetBackoff
|
||||
}
|
||||
return nil
|
||||
}
|
||||
backoff.RunF(ctx, runStreamWithBackoff, s.backoff)
|
||||
}
|
||||
|
||||
// send is a long running goroutine that handles sending discovery requests for
|
||||
// two scenarios:
|
||||
// - a new subscription or unsubscription request is received
|
||||
// - a new stream is created after the previous one failed
|
||||
func (s *adsStreamImpl) send(ctx context.Context) {
|
||||
// Stores the most recent stream instance received on streamCh.
|
||||
var stream clients.Stream
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case stream = <-s.streamCh:
|
||||
if err := s.sendExisting(stream); err != nil {
|
||||
// Send failed, clear the current stream. Attempt to resend will
|
||||
// only be made after a new stream is created.
|
||||
stream = nil
|
||||
continue
|
||||
}
|
||||
case <-s.notifySender:
|
||||
// If there's no stream yet, skip the request. This request will be resent
|
||||
// when a new stream is created. If no stream is created, the watcher will
|
||||
// timeout (same as server not sending response back).
|
||||
if stream == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Resetting the pendingRequests slice to nil works for both cases:
|
||||
// - When we successfully sends the requests out on the wire.
|
||||
// - When sending fails. This can happen only when the stream fails,
|
||||
// and in this case, we rely on the `sendExisting` to send out
|
||||
// requests for all subscriptions when the stream is recreated.
|
||||
s.mu.Lock()
|
||||
if err := s.sendNewLocked(stream, s.pendingRequests); err != nil {
|
||||
stream = nil
|
||||
}
|
||||
s.pendingRequests = nil
|
||||
s.mu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sendNewLocked attempts to send a discovery request based on a new subscription or
|
||||
// unsubscription. This method also starts the watch expiry timer for resources
|
||||
// that were sent in the request for the first time, i.e. their watch state is
|
||||
// `watchStateStarted`.
|
||||
//
|
||||
// Caller needs to hold c.mu.
|
||||
func (s *adsStreamImpl) sendNewLocked(stream clients.Stream, requests []request) error {
|
||||
for _, req := range requests {
|
||||
state := s.resourceTypeState[req.typ]
|
||||
if err := s.sendMessageLocked(stream, req.resourceNames, req.typ.TypeURL, state.version, state.nonce, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
s.startWatchTimersLocked(req.typ, req.resourceNames)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sendExisting sends out discovery requests for existing resources when
|
||||
// recovering from a broken stream.
|
||||
//
|
||||
// The stream argument is guaranteed to be non-nil.
|
||||
func (s *adsStreamImpl) sendExisting(stream clients.Stream) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
// Clear any queued requests. Previously subscribed to resources will be
|
||||
// resent below.
|
||||
s.pendingRequests = nil
|
||||
|
||||
for typ, state := range s.resourceTypeState {
|
||||
// Reset only the nonces map when the stream restarts.
|
||||
//
|
||||
// xDS spec says the following. See section:
|
||||
// https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#ack-nack-and-resource-type-instance-version
|
||||
//
|
||||
// Note that the version for a resource type is not a property of an
|
||||
// individual xDS stream but rather a property of the resources
|
||||
// themselves. If the stream becomes broken and the client creates a new
|
||||
// stream, the client’s initial request on the new stream should
|
||||
// indicate the most recent version seen by the client on the previous
|
||||
// stream
|
||||
state.nonce = ""
|
||||
|
||||
if len(state.subscribedResources) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
names := resourceNames(state.subscribedResources)
|
||||
if err := s.sendMessageLocked(stream, names, typ.TypeURL, state.version, state.nonce, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
s.startWatchTimersLocked(typ, names)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sendMessageLocked sends a discovery request to the server, populating the
|
||||
// different fields of the message with the given parameters. Returns a non-nil
|
||||
// error if the request could not be sent.
|
||||
//
|
||||
// Caller needs to hold c.mu.
|
||||
func (s *adsStreamImpl) sendMessageLocked(stream clients.Stream, names []string, url, version, nonce string, nackErr error) error {
|
||||
req := &v3discoverypb.DiscoveryRequest{
|
||||
ResourceNames: names,
|
||||
TypeUrl: url,
|
||||
VersionInfo: version,
|
||||
ResponseNonce: nonce,
|
||||
}
|
||||
|
||||
// The xDS protocol only requires that we send the node proto in the first
|
||||
// discovery request on every stream. Sending the node proto in every
|
||||
// request wastes CPU resources on the client and the server.
|
||||
if s.firstRequest.Load() {
|
||||
req.Node = s.nodeProto
|
||||
}
|
||||
|
||||
if nackErr != nil {
|
||||
req.ErrorDetail = &statuspb.Status{
|
||||
Code: int32(cpb.Code_INVALID_ARGUMENT), Message: nackErr.Error(),
|
||||
}
|
||||
}
|
||||
|
||||
msg, err := proto.Marshal(req)
|
||||
if err != nil {
|
||||
s.logger.Warningf("Failed to marshal DiscoveryRequest: %v", err)
|
||||
return err
|
||||
}
|
||||
if err := stream.Send(msg); err != nil {
|
||||
s.logger.Warningf("Sending ADS request for type %q, resources: %v, version: %q, nonce: %q failed: %v", url, names, version, nonce, err)
|
||||
return err
|
||||
}
|
||||
s.firstRequest.Store(false)
|
||||
|
||||
if s.logger.V(perRPCVerbosityLevel) {
|
||||
s.logger.Infof("ADS request sent: %v", pretty.ToJSON(req))
|
||||
} else if s.logger.V(2) {
|
||||
s.logger.Infof("ADS request sent for type %q, resources: %v, version: %q, nonce: %q", url, names, version, nonce)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// recv is responsible for receiving messages from the ADS stream.
|
||||
//
|
||||
// It performs the following actions:
|
||||
// - Waits for local flow control to be available before it receives a message
|
||||
// from the ADS stream. If an error is encountered here, it is handled by
|
||||
// the onError method which propagates the error to all watchers.
|
||||
// - Invokes the event handler's OnADSResponse method to process the message.
|
||||
// - Sends an ACK or NACK to the server based on the response.
|
||||
//
|
||||
// It returns a boolean indicating whether at least one message was received
|
||||
// from the server.
|
||||
func (s *adsStreamImpl) recv(stream clients.Stream) bool {
|
||||
msgReceived := false
|
||||
for {
|
||||
// Wait for ADS stream level flow control to be available.
|
||||
if s.fc.wait() {
|
||||
if s.logger.V(2) {
|
||||
s.logger.Infof("ADS stream stopped while waiting for flow control")
|
||||
}
|
||||
return msgReceived
|
||||
}
|
||||
|
||||
resources, url, version, nonce, err := s.recvMessage(stream)
|
||||
if err != nil {
|
||||
if !msgReceived {
|
||||
s.streamEstablished.Store(false)
|
||||
}
|
||||
s.onError(err, msgReceived)
|
||||
s.logger.Warningf("ADS stream closed: %v", err)
|
||||
return msgReceived
|
||||
}
|
||||
s.streamEstablished.Store(true)
|
||||
msgReceived = true
|
||||
|
||||
// Invoke the onResponse event handler to parse the incoming message and
|
||||
// decide whether to send an ACK or NACK.
|
||||
resp := response{
|
||||
resources: resources,
|
||||
typeURL: url,
|
||||
version: version,
|
||||
}
|
||||
var resourceNames []string
|
||||
var nackErr error
|
||||
s.fc.setPending(true)
|
||||
resourceNames, nackErr = s.eventHandler.onResponse(resp, sync.OnceFunc(func() { s.fc.setPending(false) }))
|
||||
if xdsresource.ErrType(nackErr) == xdsresource.ErrorTypeResourceTypeUnsupported {
|
||||
// A general guiding principle is that if the server sends
|
||||
// something the client didn't actually subscribe to, then the
|
||||
// client ignores it. Here, we have received a response with
|
||||
// resources of a type that we don't know about.
|
||||
//
|
||||
// Sending a NACK doesn't really seem appropriate here, since we're
|
||||
// not actually validating what the server sent and therefore don't
|
||||
// know that it's invalid. But we shouldn't ACK either, because we
|
||||
// don't know that it is valid.
|
||||
s.logger.Warningf("%v", nackErr)
|
||||
continue
|
||||
}
|
||||
|
||||
s.onRecv(stream, resourceNames, url, version, nonce, nackErr)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *adsStreamImpl) recvMessage(stream clients.Stream) (resources []*anypb.Any, url, version, nonce string, err error) {
|
||||
r, err := stream.Recv()
|
||||
if err != nil {
|
||||
return nil, "", "", "", err
|
||||
}
|
||||
var resp v3discoverypb.DiscoveryResponse
|
||||
if err := proto.Unmarshal(r, &resp); err != nil {
|
||||
s.logger.Infof("Failed to unmarshal response to DiscoveryResponse: %v", err)
|
||||
return nil, "", "", "", fmt.Errorf("unexpected message type %T", r)
|
||||
}
|
||||
if s.logger.V(perRPCVerbosityLevel) {
|
||||
s.logger.Infof("ADS response received: %v", pretty.ToJSON(&resp))
|
||||
} else if s.logger.V(2) {
|
||||
s.logger.Infof("ADS response received for type %q, version %q, nonce %q", resp.GetTypeUrl(), resp.GetVersionInfo(), resp.GetNonce())
|
||||
}
|
||||
return resp.GetResources(), resp.GetTypeUrl(), resp.GetVersionInfo(), resp.GetNonce(), nil
|
||||
}
|
||||
|
||||
// onRecv is invoked when a response is received from the server. The arguments
|
||||
// passed to this method correspond to the most recently received response.
|
||||
//
|
||||
// It performs the following actions:
|
||||
// - updates resource type specific state
|
||||
// - updates resource specific state for resources in the response
|
||||
// - sends an ACK or NACK to the server based on the response
|
||||
func (s *adsStreamImpl) onRecv(stream clients.Stream, names []string, url, version, nonce string, nackErr error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
// Lookup the resource type specific state based on the type URL.
|
||||
var typ ResourceType
|
||||
for t := range s.resourceTypeState {
|
||||
if t.TypeURL == url {
|
||||
typ = t
|
||||
break
|
||||
}
|
||||
}
|
||||
typeState, ok := s.resourceTypeState[typ]
|
||||
if !ok {
|
||||
s.logger.Warningf("ADS stream received a response for type %q, but no state exists for it", url)
|
||||
return
|
||||
}
|
||||
|
||||
// Update the resource type specific state. This includes:
|
||||
// - updating the nonce unconditionally
|
||||
// - updating the version only if the response is to be ACKed
|
||||
previousVersion := typeState.version
|
||||
typeState.nonce = nonce
|
||||
if nackErr == nil {
|
||||
typeState.version = version
|
||||
}
|
||||
|
||||
// Update the resource specific state. For all resources received as
|
||||
// part of this response that are in state `started` or `requested`,
|
||||
// this includes:
|
||||
// - setting the watch state to watchstateReceived
|
||||
// - stopping the expiry timer, if one exists
|
||||
for _, name := range names {
|
||||
rs, ok := typeState.subscribedResources[name]
|
||||
if !ok {
|
||||
s.logger.Warningf("ADS stream received a response for resource %q, but no state exists for it", name)
|
||||
continue
|
||||
}
|
||||
if ws := rs.State; ws == xdsresource.ResourceWatchStateStarted || ws == xdsresource.ResourceWatchStateRequested {
|
||||
rs.State = xdsresource.ResourceWatchStateReceived
|
||||
if rs.ExpiryTimer != nil {
|
||||
rs.ExpiryTimer.Stop()
|
||||
rs.ExpiryTimer = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Send an ACK or NACK.
|
||||
subscribedResourceNames := resourceNames(typeState.subscribedResources)
|
||||
if nackErr != nil {
|
||||
s.logger.Warningf("Sending NACK for resource type: %q, version: %q, nonce: %q, reason: %v", url, version, nonce, nackErr)
|
||||
s.sendMessageLocked(stream, subscribedResourceNames, url, previousVersion, nonce, nackErr)
|
||||
return
|
||||
}
|
||||
|
||||
if s.logger.V(2) {
|
||||
s.logger.Infof("Sending ACK for resource type: %q, version: %q, nonce: %q", url, version, nonce)
|
||||
}
|
||||
s.sendMessageLocked(stream, subscribedResourceNames, url, version, nonce, nil)
|
||||
}
|
||||
|
||||
// onError is called when an error occurs on the ADS stream. It stops any
|
||||
// outstanding resource timers and resets the watch state to started for any
|
||||
// resources that were in the requested state. It also handles the case where
|
||||
// the ADS stream was closed after receiving a response, which is not
|
||||
// considered an error.
|
||||
func (s *adsStreamImpl) onError(err error, msgReceived bool) {
|
||||
// For resources that been requested but not yet responded to by the
|
||||
// management server, stop the resource timers and reset the watch state to
|
||||
// watchStateStarted. This is because we don't want the expiry timer to be
|
||||
// running when we don't have a stream open to the management server.
|
||||
s.mu.Lock()
|
||||
for _, state := range s.resourceTypeState {
|
||||
for _, rs := range state.subscribedResources {
|
||||
if rs.State != xdsresource.ResourceWatchStateRequested {
|
||||
continue
|
||||
}
|
||||
if rs.ExpiryTimer != nil {
|
||||
rs.ExpiryTimer.Stop()
|
||||
rs.ExpiryTimer = nil
|
||||
}
|
||||
rs.State = xdsresource.ResourceWatchStateStarted
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
// Note that we do not consider it an error if the ADS stream was closed
|
||||
// after having received a response on the stream. This is because there
|
||||
// are legitimate reasons why the server may need to close the stream during
|
||||
// normal operations, such as needing to rebalance load or the underlying
|
||||
// connection hitting its max connection age limit.
|
||||
// (see [gRFC A9](https://github.com/grpc/proposal/blob/master/A9-server-side-conn-mgt.md)).
|
||||
if msgReceived {
|
||||
err = xdsresource.NewError(xdsresource.ErrTypeStreamFailedAfterRecv, err.Error())
|
||||
}
|
||||
|
||||
s.eventHandler.onStreamError(err)
|
||||
}
|
||||
|
||||
// startWatchTimersLocked starts the expiry timers for the given resource names
|
||||
// of the specified resource type. For each resource name, if the resource
|
||||
// watch state is in the "started" state, it transitions the state to
|
||||
// "requested" and starts an expiry timer. When the timer expires, the resource
|
||||
// watch state is set to "timeout" and the event handler callback is called.
|
||||
//
|
||||
// The caller must hold the s.mu lock.
|
||||
func (s *adsStreamImpl) startWatchTimersLocked(typ ResourceType, names []string) {
|
||||
typeState := s.resourceTypeState[typ]
|
||||
for _, name := range names {
|
||||
resourceState, ok := typeState.subscribedResources[name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if resourceState.State != xdsresource.ResourceWatchStateStarted {
|
||||
continue
|
||||
}
|
||||
resourceState.State = xdsresource.ResourceWatchStateRequested
|
||||
|
||||
rs := resourceState
|
||||
resourceState.ExpiryTimer = time.AfterFunc(s.watchExpiryTimeout, func() {
|
||||
s.mu.Lock()
|
||||
rs.State = xdsresource.ResourceWatchStateTimeout
|
||||
rs.ExpiryTimer = nil
|
||||
s.mu.Unlock()
|
||||
s.eventHandler.onWatchExpiry(typ, name)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *adsStreamImpl) adsResourceWatchStateForTesting(rType ResourceType, resourceName string) (xdsresource.ResourceWatchState, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
state, ok := s.resourceTypeState[rType]
|
||||
if !ok {
|
||||
return xdsresource.ResourceWatchState{}, fmt.Errorf("unknown resource type: %v", rType)
|
||||
}
|
||||
resourceState, ok := state.subscribedResources[resourceName]
|
||||
if !ok {
|
||||
return xdsresource.ResourceWatchState{}, fmt.Errorf("unknown resource name: %v", resourceName)
|
||||
}
|
||||
return *resourceState, nil
|
||||
}
|
||||
|
||||
func resourceNames(m map[string]*xdsresource.ResourceWatchState) []string {
|
||||
ret := make([]string, len(m))
|
||||
idx := 0
|
||||
for name := range m {
|
||||
ret[idx] = name
|
||||
idx++
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// adsFlowControl implements ADS stream level flow control that enables the ADS
|
||||
// stream to block the reading of the next message until the previous update is
|
||||
// consumed by all watchers.
|
||||
//
|
||||
// The lifetime of the flow control is tied to the lifetime of the stream. When
|
||||
// the stream is closed, it is the responsibility of the caller to stop the flow
|
||||
// control. This ensures that any goroutine blocked on the flow control's wait
|
||||
// method is unblocked.
|
||||
type adsFlowControl struct {
|
||||
mu sync.Mutex
|
||||
// cond is used to signal when the most recent update has been consumed, or
|
||||
// the flow control has been stopped (in which case, waiters should be
|
||||
// unblocked as well).
|
||||
cond *sync.Cond
|
||||
pending bool // indicates if the most recent update is pending consumption
|
||||
stopped bool // indicates if the ADS stream has been stopped
|
||||
}
|
||||
|
||||
// newADSFlowControl returns a new adsFlowControl.
|
||||
func newADSFlowControl() *adsFlowControl {
|
||||
fc := &adsFlowControl{}
|
||||
fc.cond = sync.NewCond(&fc.mu)
|
||||
return fc
|
||||
}
|
||||
|
||||
// stop marks the flow control as stopped and signals the condition variable to
|
||||
// unblock any goroutine waiting on it.
|
||||
func (fc *adsFlowControl) stop() {
|
||||
fc.mu.Lock()
|
||||
defer fc.mu.Unlock()
|
||||
|
||||
fc.stopped = true
|
||||
fc.cond.Broadcast()
|
||||
}
|
||||
|
||||
// setPending changes the internal state to indicate whether there is an update
|
||||
// pending consumption by all watchers. If there is no longer a pending update,
|
||||
// the condition variable is signaled to allow the recv method to proceed.
|
||||
func (fc *adsFlowControl) setPending(pending bool) {
|
||||
fc.mu.Lock()
|
||||
defer fc.mu.Unlock()
|
||||
|
||||
if fc.stopped {
|
||||
return
|
||||
}
|
||||
|
||||
fc.pending = pending
|
||||
if !pending {
|
||||
fc.cond.Broadcast()
|
||||
}
|
||||
}
|
||||
|
||||
// wait blocks until all the watchers have consumed the most recent update.
|
||||
// Returns true if the flow control was stopped while waiting, false otherwise.
|
||||
func (fc *adsFlowControl) wait() bool {
|
||||
fc.mu.Lock()
|
||||
defer fc.mu.Unlock()
|
||||
|
||||
for fc.pending && !fc.stopped {
|
||||
fc.cond.Wait()
|
||||
}
|
||||
|
||||
return fc.stopped
|
||||
}
|
||||
|
||||
func (s *adsStreamImpl) isStreamEstablished() bool {
|
||||
return s.streamEstablished.Load()
|
||||
}
|
||||
996
internal/xds/clients/xdsclient/authority.go
Normal file
996
internal/xds/clients/xdsclient/authority.go
Normal file
@@ -0,0 +1,996 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"google.golang.org/grpc/grpclog"
|
||||
igrpclog "google.golang.org/grpc/internal/grpclog"
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/syncutil"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/metrics"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
|
||||
v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
|
||||
)
|
||||
|
||||
type resourceState struct {
|
||||
watchers map[ResourceWatcher]bool // Set of watchers for this resource.
|
||||
cache ResourceData // Most recent ACKed update for this resource.
|
||||
md xdsresource.UpdateMetadata // Metadata for the most recent update.
|
||||
deletionIgnored bool // True, if resource deletion was ignored for a prior update.
|
||||
xdsChannelConfigs map[*xdsChannelWithConfig]bool // Set of xdsChannels where this resource is subscribed.
|
||||
}
|
||||
|
||||
// xdsChannelForADS is used to acquire a reference to an xdsChannel. This
|
||||
// functionality is provided by the xdsClient.
|
||||
//
|
||||
// The arguments to the function are as follows:
|
||||
// - the server config for the xdsChannel
|
||||
// - the calling authority on which a set of callbacks are invoked by the
|
||||
// xdsChannel on ADS stream events
|
||||
//
|
||||
// Returns a reference to the xdsChannel and a function to release the same. A
|
||||
// non-nil error is returned if the channel creation fails and the first two
|
||||
// return values are meaningless in this case.
|
||||
type xdsChannelForADS func(*ServerConfig, *authority) (*xdsChannel, func(), error)
|
||||
|
||||
// xdsChannelWithConfig is a struct that holds an xdsChannel and its associated
|
||||
// ServerConfig, along with a cleanup function to release the xdsChannel.
|
||||
type xdsChannelWithConfig struct {
|
||||
channel *xdsChannel
|
||||
serverConfig *ServerConfig
|
||||
cleanup func()
|
||||
}
|
||||
|
||||
// authority provides the functionality required to communicate with a
|
||||
// management server corresponding to an authority name specified in the
|
||||
// xDS client configuration.
|
||||
//
|
||||
// It holds references to one or more xdsChannels, one for each server
|
||||
// configuration in the config, to allow fallback from a primary management
|
||||
// server to a secondary management server. Authorities that contain similar
|
||||
// server configuration entries will end up sharing the xdsChannel for that
|
||||
// server configuration. The xdsChannels are owned and managed by the xdsClient.
|
||||
//
|
||||
// It also contains a cache of resource state for resources requested from
|
||||
// management server(s). This cache contains the list of registered watchers and
|
||||
// the most recent resource configuration received from the management server.
|
||||
type authority struct {
|
||||
// The following fields are initialized at creation time and are read-only
|
||||
// afterwards, and therefore don't need to be protected with a mutex.
|
||||
name string // Name of the authority from xDS client configuration.
|
||||
watcherCallbackSerializer *syncutil.CallbackSerializer // Serializer to run watcher callbacks, owned by the xDS client implementation.
|
||||
getChannelForADS xdsChannelForADS // Function to get an xdsChannel for ADS, provided by the xDS client implementation.
|
||||
xdsClientSerializer *syncutil.CallbackSerializer // Serializer to run call ins from the xDS client, owned by this authority.
|
||||
xdsClientSerializerClose func() // Function to close the above serializer.
|
||||
logger *igrpclog.PrefixLogger // Logger for this authority.
|
||||
target string // The gRPC Channel target.
|
||||
metricsReporter clients.MetricsReporter
|
||||
|
||||
// The below defined fields must only be accessed in the context of the
|
||||
// serializer callback, owned by this authority.
|
||||
|
||||
// A two level map containing the state of all the resources being watched.
|
||||
//
|
||||
// The first level map key is the ResourceType (Listener, Route etc). This
|
||||
// allows us to have a single map for all resources instead of having per
|
||||
// resource-type maps.
|
||||
//
|
||||
// The second level map key is the resource name, with the value being the
|
||||
// actual state of the resource.
|
||||
resources map[ResourceType]map[string]*resourceState
|
||||
|
||||
// An ordered list of xdsChannels corresponding to the list of server
|
||||
// configurations specified for this authority in the config. The
|
||||
// ordering specifies the order in which these channels are preferred for
|
||||
// fallback.
|
||||
xdsChannelConfigs []*xdsChannelWithConfig
|
||||
|
||||
// The current active xdsChannel. Here, active does not mean that the
|
||||
// channel has a working connection to the server. It simply points to the
|
||||
// channel that we are trying to work with, based on fallback logic.
|
||||
activeXDSChannel *xdsChannelWithConfig
|
||||
}
|
||||
|
||||
// authorityBuildOptions wraps arguments required to create a new authority.
|
||||
type authorityBuildOptions struct {
|
||||
serverConfigs []ServerConfig // Server configs for the authority
|
||||
name string // Name of the authority
|
||||
serializer *syncutil.CallbackSerializer // Callback serializer for invoking watch callbacks
|
||||
getChannelForADS xdsChannelForADS // Function to acquire a reference to an xdsChannel
|
||||
logPrefix string // Prefix for logging
|
||||
target string // Target for the gRPC Channel that owns xDS Client/Authority
|
||||
metricsReporter clients.MetricsReporter // Metrics reporter for the authority
|
||||
}
|
||||
|
||||
// newAuthority creates a new authority instance with the provided
|
||||
// configuration. The authority is responsible for managing the state of
|
||||
// resources requested from the management server, as well as acquiring and
|
||||
// releasing references to channels used to communicate with the management
|
||||
// server.
|
||||
//
|
||||
// Note that no channels to management servers are created at this time. Instead
|
||||
// a channel to the first server configuration is created when the first watch
|
||||
// is registered, and more channels are created as needed by the fallback logic.
|
||||
func newAuthority(args authorityBuildOptions) *authority {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
l := grpclog.Component("xds")
|
||||
logPrefix := args.logPrefix + fmt.Sprintf("[authority %q] ", args.name)
|
||||
ret := &authority{
|
||||
name: args.name,
|
||||
watcherCallbackSerializer: args.serializer,
|
||||
getChannelForADS: args.getChannelForADS,
|
||||
xdsClientSerializer: syncutil.NewCallbackSerializer(ctx),
|
||||
xdsClientSerializerClose: cancel,
|
||||
logger: igrpclog.NewPrefixLogger(l, logPrefix),
|
||||
resources: make(map[ResourceType]map[string]*resourceState),
|
||||
target: args.target,
|
||||
metricsReporter: args.metricsReporter,
|
||||
}
|
||||
|
||||
// Create an ordered list of xdsChannels with their server configs. The
|
||||
// actual channel to the first server configuration is created when the
|
||||
// first watch is registered, and channels to other server configurations
|
||||
// are created as needed to support fallback.
|
||||
for _, sc := range args.serverConfigs {
|
||||
ret.xdsChannelConfigs = append(ret.xdsChannelConfigs, &xdsChannelWithConfig{serverConfig: &sc})
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// adsStreamFailure is called to notify the authority about an ADS stream
|
||||
// failure on an xdsChannel to the management server identified by the provided
|
||||
// server config. The error is forwarded to all the resource watchers.
|
||||
//
|
||||
// This method is called by the xDS client implementation (on all interested
|
||||
// authorities) when a stream error is reported by an xdsChannel.
|
||||
//
|
||||
// Errors of type xdsresource.ErrTypeStreamFailedAfterRecv are ignored.
|
||||
func (a *authority) adsStreamFailure(serverConfig *ServerConfig, err error) {
|
||||
a.xdsClientSerializer.TrySchedule(func(context.Context) {
|
||||
a.handleADSStreamFailure(serverConfig, err)
|
||||
})
|
||||
}
|
||||
|
||||
// Handles ADS stream failure by invoking watch callbacks and triggering
|
||||
// fallback if the associated conditions are met.
|
||||
//
|
||||
// Only executed in the context of a serializer callback.
|
||||
func (a *authority) handleADSStreamFailure(serverConfig *ServerConfig, err error) {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Connection to server %s failed with error: %v", serverConfig, err)
|
||||
}
|
||||
|
||||
// We do not consider it an error if the ADS stream was closed after having
|
||||
// received a response on the stream. This is because there are legitimate
|
||||
// reasons why the server may need to close the stream during normal
|
||||
// operations, such as needing to rebalance load or the underlying
|
||||
// connection hitting its max connection age limit. See gRFC A57 for more
|
||||
// details.
|
||||
if xdsresource.ErrType(err) == xdsresource.ErrTypeStreamFailedAfterRecv {
|
||||
a.logger.Warningf("Watchers not notified since ADS stream failed after having received at least one response: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Two conditions need to be met for fallback to be triggered:
|
||||
// 1. There is a connectivity failure on the ADS stream, as described in
|
||||
// gRFC A57. For us, this means that the ADS stream was closed before the
|
||||
// first server response was received. We already checked that condition
|
||||
// earlier in this method.
|
||||
// 2. There is at least one watcher for a resource that is not cached.
|
||||
// Cached resources include ones that
|
||||
// - have been successfully received and can be used.
|
||||
// - are considered non-existent according to xDS Protocol Specification.
|
||||
if !a.watcherExistsForUncachedResource() {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("No watchers for uncached resources. Not triggering fallback")
|
||||
}
|
||||
// Since we are not triggering fallback, propagate the connectivity
|
||||
// error to all watchers and return early.
|
||||
a.propagateConnectivityErrorToAllWatchers(err)
|
||||
return
|
||||
}
|
||||
|
||||
// Attempt to fallback to servers with lower priority than the failing one.
|
||||
currentServerIdx := a.serverIndexForConfig(serverConfig)
|
||||
for i := currentServerIdx + 1; i < len(a.xdsChannelConfigs); i++ {
|
||||
if a.fallbackToServer(a.xdsChannelConfigs[i]) {
|
||||
// Since we have successfully triggered fallback, we don't have to
|
||||
// notify watchers about the connectivity error.
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Having exhausted all available servers, we must notify watchers of the
|
||||
// connectivity error - A71.
|
||||
a.propagateConnectivityErrorToAllWatchers(err)
|
||||
}
|
||||
|
||||
// propagateConnectivityErrorToAllWatchers propagates the given connection error
|
||||
// to all watchers of all resources.
|
||||
//
|
||||
// Only executed in the context of a serializer callback.
|
||||
func (a *authority) propagateConnectivityErrorToAllWatchers(err error) {
|
||||
for _, rType := range a.resources {
|
||||
for _, state := range rType {
|
||||
for watcher := range state.watchers {
|
||||
if state.cache == nil {
|
||||
a.watcherCallbackSerializer.TrySchedule(func(context.Context) {
|
||||
watcher.ResourceError(xdsresource.NewErrorf(xdsresource.ErrorTypeConnection, "xds: error received from xDS stream: %v", err), func() {})
|
||||
})
|
||||
} else {
|
||||
a.watcherCallbackSerializer.TrySchedule(func(context.Context) {
|
||||
watcher.AmbientError(xdsresource.NewErrorf(xdsresource.ErrorTypeConnection, "xds: error received from xDS stream: %v", err), func() {})
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// serverIndexForConfig returns the index of the xdsChannelConfig matching the
|
||||
// provided server config, panicking if no match is found (which indicates a
|
||||
// programming error).
|
||||
func (a *authority) serverIndexForConfig(sc *ServerConfig) int {
|
||||
for i, cfg := range a.xdsChannelConfigs {
|
||||
if isServerConfigEqual(sc, cfg.serverConfig) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
panic(fmt.Sprintf("no server config matching %v found", sc))
|
||||
}
|
||||
|
||||
// Determines the server to fallback to and triggers fallback to the same. If
|
||||
// required, creates an xdsChannel to that server, and re-subscribes to all
|
||||
// existing resources.
|
||||
//
|
||||
// Only executed in the context of a serializer callback.
|
||||
func (a *authority) fallbackToServer(xc *xdsChannelWithConfig) bool {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Attempting to initiate fallback to server %q", xc.serverConfig)
|
||||
}
|
||||
|
||||
if xc.channel != nil {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Channel to the next server in the list %q already exists", xc.serverConfig)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
channel, cleanup, err := a.getChannelForADS(xc.serverConfig, a)
|
||||
if err != nil {
|
||||
a.logger.Errorf("Failed to create xDS channel: %v", err)
|
||||
return false
|
||||
}
|
||||
xc.channel = channel
|
||||
xc.cleanup = cleanup
|
||||
a.activeXDSChannel = xc
|
||||
|
||||
// Subscribe to all existing resources from the new management server.
|
||||
for typ, resources := range a.resources {
|
||||
for name, state := range resources {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Resubscribing to resource of type %q and name %q", typ.TypeName, name)
|
||||
}
|
||||
xc.channel.subscribe(typ, name)
|
||||
|
||||
// Add the new channel to the list of xdsChannels from which this
|
||||
// resource has been requested from. Retain the cached resource and
|
||||
// the set of existing watchers (and other metadata fields) in the
|
||||
// resource state.
|
||||
state.xdsChannelConfigs[xc] = true
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// adsResourceUpdate is called to notify the authority about a resource update
|
||||
// received on the ADS stream.
|
||||
//
|
||||
// This method is called by the xDS client implementation (on all interested
|
||||
// authorities) when a stream error is reported by an xdsChannel.
|
||||
func (a *authority) adsResourceUpdate(serverConfig *ServerConfig, rType ResourceType, updates map[string]dataAndErrTuple, md xdsresource.UpdateMetadata, onDone func()) {
|
||||
a.xdsClientSerializer.TrySchedule(func(context.Context) {
|
||||
a.handleADSResourceUpdate(serverConfig, rType, updates, md, onDone)
|
||||
})
|
||||
}
|
||||
|
||||
// handleADSResourceUpdate processes an update from the xDS client, updating the
|
||||
// resource cache and notifying any registered watchers of the update.
|
||||
//
|
||||
// If the update is received from a higher priority xdsChannel that was
|
||||
// previously down, we revert to it and close all lower priority xdsChannels.
|
||||
//
|
||||
// Once the update has been processed by all watchers, the authority is expected
|
||||
// to invoke the onDone callback.
|
||||
//
|
||||
// Only executed in the context of a serializer callback.
|
||||
func (a *authority) handleADSResourceUpdate(serverConfig *ServerConfig, rType ResourceType, updates map[string]dataAndErrTuple, md xdsresource.UpdateMetadata, onDone func()) {
|
||||
if !a.handleRevertingToPrimaryOnUpdate(serverConfig) {
|
||||
return
|
||||
}
|
||||
|
||||
// We build a list of callback funcs to invoke, and invoke them at the end
|
||||
// of this method instead of inline (when handling the update for a
|
||||
// particular resource), because we want to make sure that all calls to
|
||||
// increment watcherCnt happen before any callbacks are invoked. This will
|
||||
// ensure that the onDone callback is never invoked before all watcher
|
||||
// callbacks are invoked, and the watchers have processed the update.
|
||||
watcherCnt := new(atomic.Int64)
|
||||
done := func() {
|
||||
if watcherCnt.Add(-1) == 0 {
|
||||
onDone()
|
||||
}
|
||||
}
|
||||
var funcsToSchedule []func(context.Context)
|
||||
defer func() {
|
||||
if len(funcsToSchedule) == 0 {
|
||||
// When there are no watchers for the resources received as part of
|
||||
// this update, invoke onDone explicitly to unblock the next read on
|
||||
// the ADS stream.
|
||||
onDone()
|
||||
return
|
||||
}
|
||||
for _, f := range funcsToSchedule {
|
||||
a.watcherCallbackSerializer.ScheduleOr(f, onDone)
|
||||
}
|
||||
}()
|
||||
|
||||
resourceStates := a.resources[rType]
|
||||
for name, uErr := range updates {
|
||||
state, ok := resourceStates[name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := uErr.Err; err != nil {
|
||||
if a.metricsReporter != nil {
|
||||
a.metricsReporter.ReportMetric(&metrics.ResourceUpdateInvalid{
|
||||
ServerURI: serverConfig.ServerIdentifier.ServerURI, ResourceType: rType.TypeName,
|
||||
})
|
||||
}
|
||||
|
||||
// Notify watchers only if this is not a duplicated error from the previous update.
|
||||
if errState := state.md.ErrState; errState == nil || errState.Err == nil || state.md.ErrState.Err.Error() != err.Error() {
|
||||
for watcher := range state.watchers {
|
||||
watcherCnt.Add(1)
|
||||
if state.cache == nil {
|
||||
funcsToSchedule = append(funcsToSchedule, func(context.Context) { watcher.ResourceError(err, done) })
|
||||
} else {
|
||||
funcsToSchedule = append(funcsToSchedule, func(context.Context) { watcher.AmbientError(err, done) })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// On error, keep previous version of the resource. But update
|
||||
// status to NACKed and capture the specific per-resource error (and
|
||||
// not the overall error for the update) in the ErrState field.
|
||||
state.md.Status = xdsresource.ServiceStatusNACKed
|
||||
state.md.ErrState = &xdsresource.UpdateErrorMetadata{
|
||||
Version: md.ErrState.Version,
|
||||
Err: err,
|
||||
Timestamp: md.ErrState.Timestamp,
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if a.metricsReporter != nil {
|
||||
a.metricsReporter.ReportMetric(&metrics.ResourceUpdateValid{
|
||||
ServerURI: serverConfig.ServerIdentifier.ServerURI, ResourceType: rType.TypeName,
|
||||
})
|
||||
}
|
||||
|
||||
if state.deletionIgnored {
|
||||
state.deletionIgnored = false
|
||||
a.logger.Infof("A valid update was received for resource %q of type %q after previously ignoring a deletion", name, rType.TypeName)
|
||||
}
|
||||
// Notify watchers if any of these conditions are met:
|
||||
// - this is the first update for this resource
|
||||
// - this update is different from the one currently cached
|
||||
// - the previous update for this resource was NACKed, but the update
|
||||
// before that was the same as this update.
|
||||
if state.cache == nil || !state.cache.Equal(uErr.Resource) || state.md.ErrState != nil {
|
||||
// Update the resource cache.
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Resource type %q with name %q added to cache", rType.TypeName, name)
|
||||
}
|
||||
state.cache = uErr.Resource
|
||||
|
||||
for watcher := range state.watchers {
|
||||
resource := uErr.Resource
|
||||
watcherCnt.Add(1)
|
||||
funcsToSchedule = append(funcsToSchedule, func(context.Context) { watcher.ResourceChanged(resource, done) })
|
||||
}
|
||||
}
|
||||
|
||||
// Set status to ACK, and clear error state. The metadata might be a
|
||||
// NACK metadata because some other resources in the same response
|
||||
// are invalid.
|
||||
state.md = md
|
||||
state.md.ErrState = nil
|
||||
state.md.Status = xdsresource.ServiceStatusACKed
|
||||
if md.ErrState != nil {
|
||||
state.md.Version = md.ErrState.Version
|
||||
}
|
||||
}
|
||||
|
||||
// If this resource type requires that all resources be present in every
|
||||
// SotW response from the server, a response that does not include a
|
||||
// previously seen resource will be interpreted as a deletion of that
|
||||
// resource unless ignore_resource_deletion option was set in the server
|
||||
// config.
|
||||
if !rType.AllResourcesRequiredInSotW {
|
||||
return
|
||||
}
|
||||
for name, state := range resourceStates {
|
||||
if state.cache == nil {
|
||||
// If the resource state does not contain a cached update, which can
|
||||
// happen when:
|
||||
// - resource was newly requested but has not yet been received, or,
|
||||
// - resource was removed as part of a previous update,
|
||||
// we don't want to generate an error for the watchers.
|
||||
//
|
||||
// For the first of the above two conditions, this ADS response may
|
||||
// be in reaction to an earlier request that did not yet request the
|
||||
// new resource, so its absence from the response does not
|
||||
// necessarily indicate that the resource does not exist. For that
|
||||
// case, we rely on the request timeout instead.
|
||||
//
|
||||
// For the second of the above two conditions, we already generated
|
||||
// an error when we received the first response which removed this
|
||||
// resource. So, there is no need to generate another one.
|
||||
continue
|
||||
}
|
||||
if _, ok := updates[name]; ok {
|
||||
// If the resource was present in the response, move on.
|
||||
continue
|
||||
}
|
||||
if state.md.Status == xdsresource.ServiceStatusNotExist {
|
||||
// The metadata status is set to "ServiceStatusNotExist" if a
|
||||
// previous update deleted this resource, in which case we do not
|
||||
// want to repeatedly call the watch callbacks with a
|
||||
// "resource-not-found" error.
|
||||
continue
|
||||
}
|
||||
if serverConfig.SupportsServerFeature(ServerFeatureIgnoreResourceDeletion) {
|
||||
// Per A53, resource deletions are ignored if the
|
||||
// `ignore_resource_deletion` server feature is enabled through the
|
||||
// xDS client configuration. If the resource deletion is to be
|
||||
// ignored, the resource is not removed from the cache and the
|
||||
// corresponding OnResourceDoesNotExist() callback is not invoked on
|
||||
// the watchers.
|
||||
if !state.deletionIgnored {
|
||||
state.deletionIgnored = true
|
||||
a.logger.Warningf("Ignoring resource deletion for resource %q of type %q", name, rType.TypeName)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// If we get here, it means that the resource exists in cache, but not
|
||||
// in the new update. Delete the resource from cache, and send a
|
||||
// resource not found error to indicate that the resource has been
|
||||
// removed. Metadata for the resource is still maintained, as this is
|
||||
// required by CSDS.
|
||||
state.cache = nil
|
||||
state.md = xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}
|
||||
for watcher := range state.watchers {
|
||||
watcherCnt.Add(1)
|
||||
funcsToSchedule = append(funcsToSchedule, func(context.Context) {
|
||||
watcher.ResourceError(xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "xds: resource %q of type %q has been removed", name, rType.TypeName), done)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// adsResourceDoesNotExist is called by the xDS client implementation (on all
|
||||
// interested authorities) to notify the authority that a subscribed resource
|
||||
// does not exist.
|
||||
func (a *authority) adsResourceDoesNotExist(rType ResourceType, resourceName string) {
|
||||
a.xdsClientSerializer.TrySchedule(func(context.Context) {
|
||||
a.handleADSResourceDoesNotExist(rType, resourceName)
|
||||
})
|
||||
}
|
||||
|
||||
// handleADSResourceDoesNotExist is called when a subscribed resource does not
|
||||
// exist. It removes the resource from the cache, updates the metadata status
|
||||
// to ServiceStatusNotExist, and notifies all watchers that the resource does
|
||||
// not exist.
|
||||
func (a *authority) handleADSResourceDoesNotExist(rType ResourceType, resourceName string) {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Watch for resource %q of type %s timed out", resourceName, rType.TypeName)
|
||||
}
|
||||
|
||||
resourceStates := a.resources[rType]
|
||||
if resourceStates == nil {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Resource %q of type %s currently not being watched", resourceName, rType.TypeName)
|
||||
}
|
||||
return
|
||||
}
|
||||
state, ok := resourceStates[resourceName]
|
||||
if !ok {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Resource %q of type %s currently not being watched", resourceName, rType.TypeName)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
state.cache = nil
|
||||
state.md = xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}
|
||||
for watcher := range state.watchers {
|
||||
watcher := watcher
|
||||
a.watcherCallbackSerializer.TrySchedule(func(context.Context) {
|
||||
watcher.ResourceError(xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "xds: resource %q of type %q has been removed", resourceName, rType.TypeName), func() {})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// handleRevertingToPrimaryOnUpdate is called when a resource update is received
|
||||
// from the xDS client.
|
||||
//
|
||||
// If the update is from the currently active server, nothing is done. Else, all
|
||||
// lower priority servers are closed and the active server is reverted to the
|
||||
// highest priority server that sent the update.
|
||||
//
|
||||
// The return value indicates whether subsequent processing of the resource
|
||||
// update should continue or not.
|
||||
//
|
||||
// This method is only executed in the context of a serializer callback.
|
||||
func (a *authority) handleRevertingToPrimaryOnUpdate(serverConfig *ServerConfig) bool {
|
||||
if a.activeXDSChannel == nil {
|
||||
// This can happen only when all watches on this authority have been
|
||||
// removed, and the xdsChannels have been closed. This update should
|
||||
// have been received prior to closing of the channel, and therefore
|
||||
// must be ignored.
|
||||
return false
|
||||
}
|
||||
|
||||
if isServerConfigEqual(serverConfig, a.activeXDSChannel.serverConfig) {
|
||||
// If the resource update is from the current active server, nothing
|
||||
// needs to be done from fallback point of view.
|
||||
return true
|
||||
}
|
||||
|
||||
// If the resource update is not from the current active server, it means
|
||||
// that we have received an update either from:
|
||||
// - a server that has a higher priority than the current active server and
|
||||
// therefore we need to revert back to it and close all lower priority
|
||||
// servers, or,
|
||||
// - a server that has a lower priority than the current active server. This
|
||||
// can happen when the server close and the response race against each
|
||||
// other. We can safely ignore this update, since we have already reverted
|
||||
// to the higher priority server, and closed all lower priority servers.
|
||||
serverIdx := a.serverIndexForConfig(serverConfig)
|
||||
activeServerIdx := a.serverIndexForConfig(a.activeXDSChannel.serverConfig)
|
||||
if activeServerIdx < serverIdx {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Ignoring update from lower priority server [%d] %q", serverIdx, serverConfig)
|
||||
}
|
||||
return false
|
||||
}
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Received update from higher priority server [%d] %q", serverIdx, serverConfig)
|
||||
}
|
||||
|
||||
// At this point, we are guaranteed that we have received a response from a
|
||||
// higher priority server compared to the current active server. So, we
|
||||
// revert to the higher priorty server and close all lower priority ones.
|
||||
a.activeXDSChannel = a.xdsChannelConfigs[serverIdx]
|
||||
|
||||
// Close all lower priority channels.
|
||||
//
|
||||
// But before closing any channel, we need to unsubscribe from any resources
|
||||
// that were subscribed to on this channel. Resources could be subscribed to
|
||||
// from multiple channels as we fallback to lower priority servers. But when
|
||||
// a higher priority one comes back up, we need to unsubscribe from all
|
||||
// lower priority ones before releasing the reference to them.
|
||||
for i := serverIdx + 1; i < len(a.xdsChannelConfigs); i++ {
|
||||
cfg := a.xdsChannelConfigs[i]
|
||||
|
||||
for rType, rState := range a.resources {
|
||||
for resourceName, state := range rState {
|
||||
for xcc := range state.xdsChannelConfigs {
|
||||
if xcc != cfg {
|
||||
continue
|
||||
}
|
||||
// If the current resource is subscribed to on this channel,
|
||||
// unsubscribe, and remove the channel from the list of
|
||||
// channels that this resource is subscribed to.
|
||||
xcc.channel.unsubscribe(rType, resourceName)
|
||||
delete(state.xdsChannelConfigs, xcc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Release the reference to the channel.
|
||||
if cfg.cleanup != nil {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Closing lower priority server [%d] %q", i, cfg.serverConfig)
|
||||
}
|
||||
cfg.cleanup()
|
||||
cfg.cleanup = nil
|
||||
}
|
||||
cfg.channel = nil
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// watchResource registers a new watcher for the specified resource type and
|
||||
// name. It returns a function that can be called to cancel the watch.
|
||||
//
|
||||
// If this is the first watch for any resource on this authority, an xdsChannel
|
||||
// to the first management server (from the list of server configurations) will
|
||||
// be created.
|
||||
//
|
||||
// If this is the first watch for the given resource name, it will subscribe to
|
||||
// the resource with the xdsChannel. If a cached copy of the resource exists, it
|
||||
// will immediately notify the new watcher. When the last watcher for a resource
|
||||
// is removed, it will unsubscribe the resource from the xdsChannel.
|
||||
func (a *authority) watchResource(rType ResourceType, resourceName string, watcher ResourceWatcher) func() {
|
||||
cleanup := func() {}
|
||||
done := make(chan struct{})
|
||||
|
||||
a.xdsClientSerializer.ScheduleOr(func(context.Context) {
|
||||
defer close(done)
|
||||
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("New watch for type %q, resource name %q", rType.TypeName, resourceName)
|
||||
}
|
||||
|
||||
xdsChannel, err := a.xdsChannelToUse()
|
||||
if err != nil {
|
||||
a.watcherCallbackSerializer.TrySchedule(func(context.Context) { watcher.ResourceError(err, func() {}) })
|
||||
return
|
||||
}
|
||||
|
||||
// Lookup the entry for the resource type in the top-level map. If there is
|
||||
// no entry for this resource type, create one.
|
||||
resources := a.resources[rType]
|
||||
if resources == nil {
|
||||
resources = make(map[string]*resourceState)
|
||||
a.resources[rType] = resources
|
||||
}
|
||||
|
||||
// Lookup the resource state for the particular resource name that the watch
|
||||
// is being registered for. If this is the first watch for this resource
|
||||
// name, request it from the management server.
|
||||
state := resources[resourceName]
|
||||
if state == nil {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("First watch for type %q, resource name %q", rType.TypeName, resourceName)
|
||||
}
|
||||
state = &resourceState{
|
||||
watchers: make(map[ResourceWatcher]bool),
|
||||
md: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusRequested},
|
||||
xdsChannelConfigs: map[*xdsChannelWithConfig]bool{xdsChannel: true},
|
||||
}
|
||||
resources[resourceName] = state
|
||||
xdsChannel.channel.subscribe(rType, resourceName)
|
||||
}
|
||||
// Always add the new watcher to the set of watchers.
|
||||
state.watchers[watcher] = true
|
||||
|
||||
// If we have a cached copy of the resource, notify the new watcher
|
||||
// immediately.
|
||||
if state.cache != nil {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Resource type %q with resource name %q found in cache: %v", rType.TypeName, resourceName, state.cache)
|
||||
}
|
||||
// state can only be accessed in the context of an
|
||||
// xdsClientSerializer callback. Hence making a copy of the cached
|
||||
// resource here for watchCallbackSerializer.
|
||||
resource := state.cache
|
||||
a.watcherCallbackSerializer.TrySchedule(func(context.Context) { watcher.ResourceChanged(resource, func() {}) })
|
||||
}
|
||||
// If last update was NACK'd, notify the new watcher of error
|
||||
// immediately as well.
|
||||
if state.md.Status == xdsresource.ServiceStatusNACKed {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Resource type %q with resource name %q was NACKed", rType.TypeName, resourceName)
|
||||
}
|
||||
// state can only be accessed in the context of an
|
||||
// xdsClientSerializer callback. Hence making a copy of the error
|
||||
// here for watchCallbackSerializer.
|
||||
err := state.md.ErrState.Err
|
||||
if state.cache == nil {
|
||||
a.watcherCallbackSerializer.TrySchedule(func(context.Context) { watcher.ResourceError(err, func() {}) })
|
||||
} else {
|
||||
a.watcherCallbackSerializer.TrySchedule(func(context.Context) { watcher.AmbientError(err, func() {}) })
|
||||
}
|
||||
}
|
||||
// If the metadata field is updated to indicate that the management
|
||||
// server does not have this resource, notify the new watcher.
|
||||
if state.md.Status == xdsresource.ServiceStatusNotExist {
|
||||
a.watcherCallbackSerializer.TrySchedule(func(context.Context) {
|
||||
watcher.ResourceError(xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "xds: resource %q of type %q has been removed", resourceName, rType.TypeName), func() {})
|
||||
})
|
||||
}
|
||||
cleanup = a.unwatchResource(rType, resourceName, watcher)
|
||||
}, func() {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Failed to schedule a watch for type %q, resource name %q, because the xDS client is closed", rType.TypeName, resourceName)
|
||||
}
|
||||
close(done)
|
||||
})
|
||||
<-done
|
||||
return cleanup
|
||||
}
|
||||
|
||||
func (a *authority) unwatchResource(rType ResourceType, resourceName string, watcher ResourceWatcher) func() {
|
||||
return sync.OnceFunc(func() {
|
||||
done := make(chan struct{})
|
||||
a.xdsClientSerializer.ScheduleOr(func(context.Context) {
|
||||
defer close(done)
|
||||
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Canceling a watch for type %q, resource name %q", rType.TypeName, resourceName)
|
||||
}
|
||||
|
||||
// Lookup the resource type from the resource cache. The entry is
|
||||
// guaranteed to be present, since *we* were the ones who added it in
|
||||
// there when the watch was registered.
|
||||
resources := a.resources[rType]
|
||||
state := resources[resourceName]
|
||||
|
||||
// Delete this particular watcher from the list of watchers, so that its
|
||||
// callback will not be invoked in the future.
|
||||
delete(state.watchers, watcher)
|
||||
if len(state.watchers) > 0 {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Other watchers exist for type %q, resource name %q", rType.TypeName, resourceName)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// There are no more watchers for this resource. Unsubscribe this
|
||||
// resource from all channels where it was subscribed to and delete
|
||||
// the state associated with it.
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Removing last watch for resource name %q", resourceName)
|
||||
}
|
||||
for xcc := range state.xdsChannelConfigs {
|
||||
xcc.channel.unsubscribe(rType, resourceName)
|
||||
}
|
||||
delete(resources, resourceName)
|
||||
|
||||
// If there are no more watchers for this resource type, delete the
|
||||
// resource type from the top-level map.
|
||||
if len(resources) == 0 {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Removing last watch for resource type %q", rType.TypeName)
|
||||
}
|
||||
delete(a.resources, rType)
|
||||
}
|
||||
// If there are no more watchers for any resource type, release the
|
||||
// reference to the xdsChannels.
|
||||
if len(a.resources) == 0 {
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Removing last watch for any resource type, releasing reference to the xdsChannel")
|
||||
}
|
||||
a.closeXDSChannels()
|
||||
}
|
||||
}, func() { close(done) })
|
||||
<-done
|
||||
})
|
||||
}
|
||||
|
||||
// xdsChannelToUse returns the xdsChannel to use for communicating with the
|
||||
// management server. If an active channel is available, it returns that.
|
||||
// Otherwise, it creates a new channel using the first server configuration in
|
||||
// the list of configurations, and returns that.
|
||||
//
|
||||
// A non-nil error is returned if the channel creation fails.
|
||||
//
|
||||
// Only executed in the context of a serializer callback.
|
||||
func (a *authority) xdsChannelToUse() (*xdsChannelWithConfig, error) {
|
||||
if a.activeXDSChannel != nil {
|
||||
return a.activeXDSChannel, nil
|
||||
}
|
||||
|
||||
sc := a.xdsChannelConfigs[0].serverConfig
|
||||
xc, cleanup, err := a.getChannelForADS(sc, a)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.xdsChannelConfigs[0].channel = xc
|
||||
a.xdsChannelConfigs[0].cleanup = cleanup
|
||||
a.activeXDSChannel = a.xdsChannelConfigs[0]
|
||||
return a.activeXDSChannel, nil
|
||||
}
|
||||
|
||||
// closeXDSChannels closes all the xDS channels associated with this authority,
|
||||
// when there are no more watchers for any resource type.
|
||||
//
|
||||
// Only executed in the context of a serializer callback.
|
||||
func (a *authority) closeXDSChannels() {
|
||||
for _, xcc := range a.xdsChannelConfigs {
|
||||
if xcc.cleanup != nil {
|
||||
xcc.cleanup()
|
||||
xcc.cleanup = nil
|
||||
}
|
||||
xcc.channel = nil
|
||||
}
|
||||
a.activeXDSChannel = nil
|
||||
}
|
||||
|
||||
// watcherExistsForUncachedResource returns true if there is at least one
|
||||
// watcher for a resource that has not yet been cached.
|
||||
//
|
||||
// Only executed in the context of a serializer callback.
|
||||
func (a *authority) watcherExistsForUncachedResource() bool {
|
||||
for _, resourceStates := range a.resources {
|
||||
for _, state := range resourceStates {
|
||||
if state.md.Status == xdsresource.ServiceStatusRequested {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// dumpResources returns a dump of the resource configuration cached by this
|
||||
// authority, for CSDS purposes.
|
||||
func (a *authority) dumpResources() []*v3statuspb.ClientConfig_GenericXdsConfig {
|
||||
var ret []*v3statuspb.ClientConfig_GenericXdsConfig
|
||||
done := make(chan struct{})
|
||||
|
||||
a.xdsClientSerializer.ScheduleOr(func(context.Context) {
|
||||
defer close(done)
|
||||
ret = a.resourceConfig()
|
||||
}, func() { close(done) })
|
||||
<-done
|
||||
return ret
|
||||
}
|
||||
|
||||
// resourceConfig returns a slice of GenericXdsConfig objects representing the
|
||||
// current state of all resources managed by this authority. This is used for
|
||||
// reporting the current state of the xDS client.
|
||||
//
|
||||
// Only executed in the context of a serializer callback.
|
||||
func (a *authority) resourceConfig() []*v3statuspb.ClientConfig_GenericXdsConfig {
|
||||
var ret []*v3statuspb.ClientConfig_GenericXdsConfig
|
||||
for rType, resourceStates := range a.resources {
|
||||
typeURL := rType.TypeURL
|
||||
for name, state := range resourceStates {
|
||||
var raw *anypb.Any
|
||||
if state.cache != nil {
|
||||
raw = &anypb.Any{TypeUrl: typeURL, Value: state.cache.Bytes()}
|
||||
}
|
||||
config := &v3statuspb.ClientConfig_GenericXdsConfig{
|
||||
TypeUrl: typeURL,
|
||||
Name: name,
|
||||
VersionInfo: state.md.Version,
|
||||
XdsConfig: raw,
|
||||
LastUpdated: timestamppb.New(state.md.Timestamp),
|
||||
ClientStatus: serviceStatusToProto(state.md.Status),
|
||||
}
|
||||
if errState := state.md.ErrState; errState != nil {
|
||||
config.ErrorState = &v3adminpb.UpdateFailureState{
|
||||
LastUpdateAttempt: timestamppb.New(errState.Timestamp),
|
||||
Details: errState.Err.Error(),
|
||||
VersionInfo: errState.Version,
|
||||
}
|
||||
}
|
||||
ret = append(ret, config)
|
||||
}
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func (a *authority) close() {
|
||||
a.xdsClientSerializerClose()
|
||||
<-a.xdsClientSerializer.Done()
|
||||
if a.logger.V(2) {
|
||||
a.logger.Infof("Closed")
|
||||
}
|
||||
}
|
||||
|
||||
func serviceStatusToProto(serviceStatus xdsresource.ServiceStatus) v3adminpb.ClientResourceStatus {
|
||||
switch serviceStatus {
|
||||
case xdsresource.ServiceStatusUnknown:
|
||||
return v3adminpb.ClientResourceStatus_UNKNOWN
|
||||
case xdsresource.ServiceStatusRequested:
|
||||
return v3adminpb.ClientResourceStatus_REQUESTED
|
||||
case xdsresource.ServiceStatusNotExist:
|
||||
return v3adminpb.ClientResourceStatus_DOES_NOT_EXIST
|
||||
case xdsresource.ServiceStatusACKed:
|
||||
return v3adminpb.ClientResourceStatus_ACKED
|
||||
case xdsresource.ServiceStatusNACKed:
|
||||
return v3adminpb.ClientResourceStatus_NACKED
|
||||
default:
|
||||
return v3adminpb.ClientResourceStatus_UNKNOWN
|
||||
}
|
||||
}
|
||||
|
||||
func (a *authority) resourceWatchStateForTesting(rType ResourceType, resourceName string) (state xdsresource.ResourceWatchState, err error) {
|
||||
done := make(chan struct{})
|
||||
a.xdsClientSerializer.ScheduleOr(func(context.Context) {
|
||||
state, err = a.activeXDSChannel.channel.ads.adsResourceWatchStateForTesting(rType, resourceName)
|
||||
close(done)
|
||||
}, func() {
|
||||
err = errors.New("failed to retrieve resource watch state because the xDS client is closed")
|
||||
close(done)
|
||||
})
|
||||
<-done
|
||||
|
||||
return state, err
|
||||
}
|
||||
|
||||
// resourceStats returns a snapshot of the current state of all resources watched
|
||||
// by this authority. The return value is a nested map where:
|
||||
// - The outer map's key is the resource type name (e.g., "ListenerResource").
|
||||
// - The inner map's key is the cache state of the resource (e.g., "requested",
|
||||
// "acked", "nacked", "does_not_exist").
|
||||
// - The inner map's value is the total count of resources in that specific state.
|
||||
func (a *authority) resourceStats() map[string]map[string]int {
|
||||
ret := make(chan map[string]map[string]int, 1)
|
||||
op := func(context.Context) {
|
||||
summary := make(map[string]map[string]int)
|
||||
for rType, resourceMap := range a.resources {
|
||||
typeName := rType.TypeName
|
||||
if _, ok := summary[typeName]; !ok {
|
||||
summary[typeName] = make(map[string]int)
|
||||
}
|
||||
for _, state := range resourceMap {
|
||||
s := cacheState(state)
|
||||
summary[typeName][s]++
|
||||
}
|
||||
}
|
||||
|
||||
ret <- summary
|
||||
}
|
||||
a.xdsClientSerializer.ScheduleOr(op, func() {
|
||||
ret <- nil
|
||||
})
|
||||
|
||||
return <-ret
|
||||
}
|
||||
|
||||
// cacheState determines the metrics label string for a given resource state.
|
||||
func cacheState(r *resourceState) string {
|
||||
switch r.md.Status {
|
||||
case xdsresource.ServiceStatusRequested:
|
||||
return "requested"
|
||||
case xdsresource.ServiceStatusNotExist:
|
||||
return "does_not_exist"
|
||||
case xdsresource.ServiceStatusACKed:
|
||||
return "acked"
|
||||
case xdsresource.ServiceStatusNACKed:
|
||||
// If the status is NACKed, it means the *latest* update failed.
|
||||
// However, if 'r.cache' is not nil, it means we are still holding onto
|
||||
// a previously ACKed version of the resource.
|
||||
if r.cache != nil {
|
||||
return "nacked_but_cached"
|
||||
}
|
||||
return "nacked"
|
||||
default:
|
||||
// Fallback for initialization states
|
||||
return "requested"
|
||||
}
|
||||
}
|
||||
329
internal/xds/clients/xdsclient/channel.go
Normal file
329
internal/xds/clients/xdsclient/channel.go
Normal file
@@ -0,0 +1,329 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc/grpclog"
|
||||
"google.golang.org/grpc/internal/envconfig"
|
||||
igrpclog "google.golang.org/grpc/internal/grpclog"
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/backoff"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/syncutil"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
)
|
||||
|
||||
const (
|
||||
clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning"
|
||||
clientFeatureResourceWrapper = "xds.config.resource-in-sotw"
|
||||
)
|
||||
|
||||
// xdsChannelEventHandler wraps callbacks used to notify the xDS client about
|
||||
// events on the xdsChannel. Methods in this interface may be invoked
|
||||
// concurrently and the xDS client implementation needs to handle them in a
|
||||
// thread-safe manner.
|
||||
type xdsChannelEventHandler interface {
|
||||
// adsStreamFailure is called when the xdsChannel encounters an ADS stream
|
||||
// failure.
|
||||
adsStreamFailure(error)
|
||||
|
||||
// adsResourceUpdate is called when the xdsChannel receives an ADS response
|
||||
// from the xDS management server. The callback is provided with the
|
||||
// following:
|
||||
// - the resource type of the resources in the response
|
||||
// - a map of resources in the response, keyed by resource name
|
||||
// - the metadata associated with the response
|
||||
// - a callback to be invoked when the updated is processed
|
||||
adsResourceUpdate(ResourceType, map[string]dataAndErrTuple, xdsresource.UpdateMetadata, func())
|
||||
|
||||
// adsResourceDoesNotExist is called when the xdsChannel determines that a
|
||||
// requested ADS resource does not exist.
|
||||
adsResourceDoesNotExist(ResourceType, string)
|
||||
}
|
||||
|
||||
// xdsChannelOpts holds the options for creating a new xdsChannel.
|
||||
type xdsChannelOpts struct {
|
||||
transport clients.Transport // Takes ownership of this transport.
|
||||
serverConfig *ServerConfig // Configuration of the server to connect to.
|
||||
clientConfig *Config // Complete xDS client configuration, used to decode resources.
|
||||
eventHandler xdsChannelEventHandler // Callbacks for ADS stream events.
|
||||
backoff func(int) time.Duration // Backoff function to use for stream retries. Defaults to exponential backoff, if unset.
|
||||
watchExpiryTimeout time.Duration // Timeout for ADS resource watch expiry.
|
||||
logPrefix string // Prefix to use for logging.
|
||||
}
|
||||
|
||||
// newXDSChannel creates a new xdsChannel instance with the provided options.
|
||||
// It performs basic validation on the provided options and initializes the
|
||||
// xdsChannel with the necessary components.
|
||||
func newXDSChannel(opts xdsChannelOpts) (*xdsChannel, error) {
|
||||
switch {
|
||||
case opts.transport == nil:
|
||||
return nil, errors.New("xdsclient: transport is nil")
|
||||
case opts.serverConfig == nil:
|
||||
return nil, errors.New("xdsclient: serverConfig is nil")
|
||||
case opts.clientConfig == nil:
|
||||
return nil, errors.New("xdsclient: clientConfig is nil")
|
||||
case opts.eventHandler == nil:
|
||||
return nil, errors.New("xdsclient: eventHandler is nil")
|
||||
}
|
||||
|
||||
xc := &xdsChannel{
|
||||
transport: opts.transport,
|
||||
serverConfig: opts.serverConfig,
|
||||
clientConfig: opts.clientConfig,
|
||||
eventHandler: opts.eventHandler,
|
||||
closed: syncutil.NewEvent(),
|
||||
}
|
||||
|
||||
l := grpclog.Component("xds")
|
||||
logPrefix := opts.logPrefix + fmt.Sprintf("[xds-channel %p] ", xc)
|
||||
xc.logger = igrpclog.NewPrefixLogger(l, logPrefix)
|
||||
|
||||
if opts.backoff == nil {
|
||||
opts.backoff = backoff.DefaultExponential.Backoff
|
||||
}
|
||||
np := internal.NodeProto(opts.clientConfig.Node)
|
||||
np.ClientFeatures = []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper}
|
||||
xc.ads = newADSStreamImpl(adsStreamOpts{
|
||||
transport: opts.transport,
|
||||
eventHandler: xc,
|
||||
backoff: opts.backoff,
|
||||
nodeProto: np,
|
||||
watchExpiryTimeout: opts.watchExpiryTimeout,
|
||||
logPrefix: logPrefix,
|
||||
})
|
||||
if xc.logger.V(2) {
|
||||
xc.logger.Infof("xdsChannel is created for ServerConfig %v", opts.serverConfig)
|
||||
}
|
||||
return xc, nil
|
||||
}
|
||||
|
||||
// xdsChannel represents a client channel to a management server, and is
|
||||
// responsible for managing the lifecycle of the ADS and LRS streams. It invokes
|
||||
// callbacks on the registered event handler for various ADS stream events.
|
||||
//
|
||||
// It is safe for concurrent use.
|
||||
type xdsChannel struct {
|
||||
// The following fields are initialized at creation time and are read-only
|
||||
// after that, and hence need not be guarded by a mutex.
|
||||
transport clients.Transport // Takes ownership of this transport (used to make streaming calls).
|
||||
ads *adsStreamImpl // An ADS stream to the management server.
|
||||
serverConfig *ServerConfig // Configuration of the server to connect to.
|
||||
clientConfig *Config // Complete xDS client configuration, used to decode resources.
|
||||
eventHandler xdsChannelEventHandler // Callbacks for ADS stream events.
|
||||
logger *igrpclog.PrefixLogger // Logger to use for logging.
|
||||
closed *syncutil.Event // Fired when the channel is closed.
|
||||
}
|
||||
|
||||
func (xc *xdsChannel) close() {
|
||||
xc.closed.Fire()
|
||||
xc.ads.Stop()
|
||||
xc.transport.Close()
|
||||
xc.logger.Infof("Shutdown")
|
||||
}
|
||||
|
||||
// subscribe adds a subscription for the given resource name of the given
|
||||
// resource type on the ADS stream.
|
||||
func (xc *xdsChannel) subscribe(typ ResourceType, name string) {
|
||||
if xc.closed.HasFired() {
|
||||
if xc.logger.V(2) {
|
||||
xc.logger.Infof("Attempt to subscribe to an xDS resource of type %s and name %q on a closed channel", typ.TypeName, name)
|
||||
}
|
||||
return
|
||||
}
|
||||
xc.ads.subscribe(typ, name)
|
||||
}
|
||||
|
||||
// unsubscribe removes the subscription for the given resource name of the given
|
||||
// resource type from the ADS stream.
|
||||
func (xc *xdsChannel) unsubscribe(typ ResourceType, name string) {
|
||||
if xc.closed.HasFired() {
|
||||
if xc.logger.V(2) {
|
||||
xc.logger.Infof("Attempt to unsubscribe to an xDS resource of type %s and name %q on a closed channel", typ.TypeName, name)
|
||||
}
|
||||
return
|
||||
}
|
||||
xc.ads.unsubscribe(typ, name)
|
||||
}
|
||||
|
||||
// The following onADSXxx() methods implement the StreamEventHandler interface
|
||||
// and are invoked by the ADS stream implementation.
|
||||
|
||||
// onStreamError is invoked when an error occurs on the ADS stream. It
|
||||
// propagates the update to the xDS client.
|
||||
func (xc *xdsChannel) onStreamError(err error) {
|
||||
if xc.closed.HasFired() {
|
||||
if xc.logger.V(2) {
|
||||
xc.logger.Infof("Received ADS stream error on a closed xdsChannel: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
xc.eventHandler.adsStreamFailure(err)
|
||||
}
|
||||
|
||||
// onWatchExpiry is invoked when a watch for a resource expires. It
|
||||
// propagates the update to the xDS client.
|
||||
func (xc *xdsChannel) onWatchExpiry(typ ResourceType, name string) {
|
||||
if xc.closed.HasFired() {
|
||||
if xc.logger.V(2) {
|
||||
xc.logger.Infof("Received ADS resource watch expiry for resource %q on a closed xdsChannel", name)
|
||||
}
|
||||
return
|
||||
}
|
||||
xc.eventHandler.adsResourceDoesNotExist(typ, name)
|
||||
}
|
||||
|
||||
// onResponse is invoked when a response is received on the ADS stream. It
|
||||
// decodes the resources in the response, and propagates the updates to the xDS
|
||||
// client.
|
||||
//
|
||||
// It returns the list of resource names in the response and any errors
|
||||
// encountered during decoding.
|
||||
func (xc *xdsChannel) onResponse(resp response, onDone func()) ([]string, error) {
|
||||
if xc.closed.HasFired() {
|
||||
if xc.logger.V(2) {
|
||||
xc.logger.Infof("Received an update from the ADS stream on closed ADS stream")
|
||||
}
|
||||
return nil, errors.New("xdsChannel is closed")
|
||||
}
|
||||
|
||||
// Lookup the resource parser based on the resource type.
|
||||
rType, ok := xc.clientConfig.ResourceTypes[resp.typeURL]
|
||||
if !ok {
|
||||
return nil, xdsresource.NewErrorf(xdsresource.ErrorTypeResourceTypeUnsupported, "Resource type URL %q unknown in response from server", resp.typeURL)
|
||||
}
|
||||
|
||||
updates, md, err := xc.decodeResponse(&rType, resp)
|
||||
var names []string
|
||||
for name := range updates {
|
||||
names = append(names, name)
|
||||
}
|
||||
|
||||
xc.eventHandler.adsResourceUpdate(rType, updates, md, onDone)
|
||||
return names, err
|
||||
}
|
||||
|
||||
// decodeResponse decodes the resources in the given ADS response.
|
||||
//
|
||||
// The opts parameter provides configuration options for decoding the resources.
|
||||
// The rType parameter specifies the resource type parser to use for decoding
|
||||
// the resources.
|
||||
//
|
||||
// The returned map contains a key for each resource in the response, with the
|
||||
// value being either the decoded resource data or an error if decoding failed.
|
||||
// The returned metadata includes the version of the response, the timestamp of
|
||||
// the update, and the status of the update (ACKed or NACKed).
|
||||
//
|
||||
// If there are any errors decoding the resources, the metadata will indicate
|
||||
// that the update was NACKed, and the returned error will contain information
|
||||
// about all errors encountered by this function.
|
||||
func (xc *xdsChannel) decodeResponse(rType *ResourceType, resp response) (map[string]dataAndErrTuple, xdsresource.UpdateMetadata, error) {
|
||||
timestamp := time.Now()
|
||||
md := xdsresource.UpdateMetadata{
|
||||
Version: resp.version,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
opts := &DecodeOptions{
|
||||
Config: xc.clientConfig,
|
||||
ServerConfig: xc.serverConfig,
|
||||
}
|
||||
|
||||
topLevelErrors := make([]error, 0) // Tracks deserialization errors, where we don't have a resource name.
|
||||
perResourceErrors := make(map[string]error) // Tracks resource validation errors, where we have a resource name.
|
||||
ret := make(map[string]dataAndErrTuple) // Return result, a map from resource name to either resource data or error.
|
||||
for _, r := range resp.resources {
|
||||
result, err := func() (res *DecodeResult, err error) {
|
||||
defer func() {
|
||||
if envconfig.XDSRecoverPanicInResourceParsing {
|
||||
if p := recover(); p != nil {
|
||||
err = fmt.Errorf("recovered from panic during resource parsing, resource: %v, panic: %v", r, p)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return rType.Decoder.Decode(NewAnyProto(r), *opts)
|
||||
}()
|
||||
|
||||
// Name field of the result is left unpopulated only when resource
|
||||
// deserialization fails.
|
||||
name := ""
|
||||
if result == nil && err == nil {
|
||||
xc.logger.Errorf("Decode() returned nil result and nil error for resource: %v", r)
|
||||
continue
|
||||
}
|
||||
if result != nil {
|
||||
name = xdsresource.ParseName(result.Name).String()
|
||||
}
|
||||
if err == nil {
|
||||
ret[name] = dataAndErrTuple{Resource: result.Resource}
|
||||
continue
|
||||
}
|
||||
if name == "" {
|
||||
topLevelErrors = append(topLevelErrors, err)
|
||||
continue
|
||||
}
|
||||
perResourceErrors[name] = err
|
||||
// Add place holder in the map so we know this resource name was in
|
||||
// the response.
|
||||
ret[name] = dataAndErrTuple{Err: xdsresource.NewError(xdsresource.ErrorTypeNACKed, err.Error())}
|
||||
}
|
||||
|
||||
if len(topLevelErrors) == 0 && len(perResourceErrors) == 0 {
|
||||
md.Status = xdsresource.ServiceStatusACKed
|
||||
return ret, md, nil
|
||||
}
|
||||
|
||||
md.Status = xdsresource.ServiceStatusNACKed
|
||||
errRet := combineErrors(rType.TypeName, topLevelErrors, perResourceErrors)
|
||||
md.ErrState = &xdsresource.UpdateErrorMetadata{
|
||||
Version: resp.version,
|
||||
Err: xdsresource.NewError(xdsresource.ErrorTypeNACKed, errRet.Error()),
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
return ret, md, errRet
|
||||
}
|
||||
|
||||
func combineErrors(rType string, topLevelErrors []error, perResourceErrors map[string]error) error {
|
||||
var errStrB strings.Builder
|
||||
errStrB.WriteString(fmt.Sprintf("error parsing %q response: ", rType))
|
||||
if len(topLevelErrors) > 0 {
|
||||
errStrB.WriteString("top level errors: ")
|
||||
for i, err := range topLevelErrors {
|
||||
if i != 0 {
|
||||
errStrB.WriteString(";\n")
|
||||
}
|
||||
errStrB.WriteString(err.Error())
|
||||
}
|
||||
}
|
||||
if len(perResourceErrors) > 0 {
|
||||
var i int
|
||||
for name, err := range perResourceErrors {
|
||||
if i != 0 {
|
||||
errStrB.WriteString(";\n")
|
||||
}
|
||||
i++
|
||||
errStrB.WriteString(fmt.Sprintf("resource %q: %v", name, err.Error()))
|
||||
}
|
||||
}
|
||||
return errors.New(errStrB.String())
|
||||
}
|
||||
821
internal/xds/clients/xdsclient/channel_test.go
Normal file
821
internal/xds/clients/xdsclient/channel_test.go
Normal file
@@ -0,0 +1,821 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/google/uuid"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/internal/envconfig"
|
||||
"google.golang.org/grpc/internal/testutils"
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
"google.golang.org/grpc/internal/xds/clients/grpctransport"
|
||||
xdstestutils "google.golang.org/grpc/internal/xds/clients/internal/testutils"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils/fakeserver"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
"google.golang.org/protobuf/testing/protocmp"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
|
||||
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
|
||||
v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
|
||||
v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
|
||||
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
|
||||
)
|
||||
|
||||
// xdsChannelForTest creates an xdsChannel to the specified serverURI for
|
||||
// testing purposes.
|
||||
func xdsChannelForTest(t *testing.T, serverURI, nodeID string, watchExpiryTimeout time.Duration) *xdsChannel {
|
||||
t.Helper()
|
||||
|
||||
// Create a grpc transport to the above management server.
|
||||
si := clients.ServerIdentifier{
|
||||
ServerURI: serverURI,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
tr, err := (grpctransport.NewBuilder(configs)).Build(si)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create a transport for server config %v: %v", si, err)
|
||||
}
|
||||
|
||||
serverCfg := ServerConfig{
|
||||
ServerIdentifier: si,
|
||||
}
|
||||
clientConfig := Config{
|
||||
Servers: []ServerConfig{serverCfg},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
ResourceTypes: map[string]ResourceType{xdsresource.V3ListenerURL: listenerType},
|
||||
}
|
||||
// Create an xdsChannel that uses everything set up above.
|
||||
xc, err := newXDSChannel(xdsChannelOpts{
|
||||
transport: tr,
|
||||
serverConfig: &serverCfg,
|
||||
clientConfig: &clientConfig,
|
||||
eventHandler: newTestEventHandler(),
|
||||
watchExpiryTimeout: watchExpiryTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xdsChannel: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { xc.close() })
|
||||
return xc
|
||||
}
|
||||
|
||||
// verifyUpdateAndMetadata verifies that the event handler received the expected
|
||||
// updates and metadata. It checks that the received resource type matches the
|
||||
// expected type, and that the received updates and metadata match the expected
|
||||
// values. The function ignores the timestamp fields in the metadata, as those
|
||||
// are expected to be different.
|
||||
func verifyUpdateAndMetadata(ctx context.Context, t *testing.T, eh *testEventHandler, wantUpdates map[string]dataAndErrTuple, wantMD xdsresource.UpdateMetadata) {
|
||||
t.Helper()
|
||||
|
||||
gotTyp, gotUpdates, gotMD, err := eh.waitForUpdate(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Timeout when waiting for update callback to be invoked on the event handler")
|
||||
}
|
||||
|
||||
if gotTyp != listenerType {
|
||||
t.Fatalf("Got resource type %v, want %v", gotTyp, listenerType)
|
||||
}
|
||||
opts := cmp.Options{
|
||||
protocmp.Transform(),
|
||||
cmpopts.EquateEmpty(),
|
||||
cmpopts.EquateErrors(),
|
||||
cmpopts.IgnoreFields(xdsresource.UpdateMetadata{}, "Timestamp"),
|
||||
cmpopts.IgnoreFields(xdsresource.UpdateErrorMetadata{}, "Timestamp"),
|
||||
}
|
||||
if diff := cmp.Diff(wantUpdates, gotUpdates, opts); diff != "" {
|
||||
t.Fatalf("Got unexpected diff in update (-want +got):\n%s\n want: %+v\n got: %+v", diff, wantUpdates, gotUpdates)
|
||||
}
|
||||
if diff := cmp.Diff(wantMD, gotMD, opts); diff != "" {
|
||||
t.Fatalf("Got unexpected diff in update (-want +got):\n%s\n want: %v\n got: %v", diff, wantMD, gotMD)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests different failure cases when creating a new xdsChannel. It checks that
|
||||
// the xdsChannel creation fails when any of the required options (transport,
|
||||
// serverConfig, bootstrapConfig, or resourceTypeGetter) are missing or nil.
|
||||
func (s) TestChannel_New_FailureCases(t *testing.T) {
|
||||
type fakeTransport struct {
|
||||
clients.Transport
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
opts xdsChannelOpts
|
||||
wantErrStr string
|
||||
}{
|
||||
{
|
||||
name: "emptyTransport",
|
||||
opts: xdsChannelOpts{},
|
||||
wantErrStr: "transport is nil",
|
||||
},
|
||||
{
|
||||
name: "emptyServerConfig",
|
||||
opts: xdsChannelOpts{transport: &fakeTransport{}},
|
||||
wantErrStr: "serverConfig is nil",
|
||||
},
|
||||
{
|
||||
name: "emptyCConfig",
|
||||
opts: xdsChannelOpts{
|
||||
transport: &fakeTransport{},
|
||||
serverConfig: &ServerConfig{},
|
||||
},
|
||||
wantErrStr: "clientConfig is nil",
|
||||
},
|
||||
{
|
||||
name: "emptyEventHandler",
|
||||
opts: xdsChannelOpts{
|
||||
transport: &fakeTransport{},
|
||||
serverConfig: &ServerConfig{},
|
||||
clientConfig: &Config{},
|
||||
},
|
||||
wantErrStr: "eventHandler is nil",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if _, err := newXDSChannel(test.opts); err == nil || !strings.Contains(err.Error(), test.wantErrStr) {
|
||||
t.Fatalf("newXDSChannel() = %v, want %q", err, test.wantErrStr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Tests different scenarios of the xdsChannel receiving a response from the
|
||||
// management server. In all scenarios, the xdsChannel is expected to pass the
|
||||
// received responses as-is to the resource parsing functionality specified by
|
||||
// the resourceTypeGetter.
|
||||
func (s) TestChannel_ADS_HandleResponseFromManagementServer(t *testing.T) {
|
||||
const (
|
||||
listenerName1 = "listener-name-1"
|
||||
listenerName2 = "listener-name-2"
|
||||
routeName = "route-name"
|
||||
clusterName = "cluster-name"
|
||||
)
|
||||
var (
|
||||
badlyMarshaledResource = &anypb.Any{
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
Value: []byte{1, 2, 3, 4},
|
||||
}
|
||||
apiListener = &v3listenerpb.ApiListener{
|
||||
ApiListener: xdstestutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
|
||||
RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
|
||||
RouteConfig: &v3routepb.RouteConfiguration{
|
||||
Name: routeName},
|
||||
},
|
||||
}),
|
||||
}
|
||||
listener1 = xdstestutils.MarshalAny(t, &v3listenerpb.Listener{
|
||||
Name: listenerName1,
|
||||
ApiListener: apiListener,
|
||||
})
|
||||
listener2 = xdstestutils.MarshalAny(t, &v3listenerpb.Listener{
|
||||
Name: listenerName2,
|
||||
ApiListener: apiListener,
|
||||
})
|
||||
)
|
||||
|
||||
tests := []struct {
|
||||
desc string
|
||||
resourceNamesToRequest []string
|
||||
managementServerResponse *v3discoverypb.DiscoveryResponse
|
||||
wantUpdates map[string]dataAndErrTuple
|
||||
wantMD xdsresource.UpdateMetadata
|
||||
wantErr error
|
||||
}{
|
||||
{
|
||||
desc: "one bad resource - deserialization failure",
|
||||
resourceNamesToRequest: []string{listenerName1},
|
||||
managementServerResponse: &v3discoverypb.DiscoveryResponse{
|
||||
VersionInfo: "0",
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
Resources: []*anypb.Any{badlyMarshaledResource},
|
||||
},
|
||||
wantUpdates: nil, // No updates expected as the response runs into unmarshaling errors.
|
||||
wantMD: xdsresource.UpdateMetadata{
|
||||
Status: xdsresource.ServiceStatusNACKed,
|
||||
Version: "0",
|
||||
ErrState: &xdsresource.UpdateErrorMetadata{
|
||||
Version: "0",
|
||||
Err: cmpopts.AnyError,
|
||||
},
|
||||
},
|
||||
wantErr: cmpopts.AnyError,
|
||||
},
|
||||
{
|
||||
desc: "one bad resource - validation failure",
|
||||
resourceNamesToRequest: []string{listenerName1},
|
||||
managementServerResponse: &v3discoverypb.DiscoveryResponse{
|
||||
VersionInfo: "0",
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
Resources: []*anypb.Any{xdstestutils.MarshalAny(t, &v3listenerpb.Listener{
|
||||
Name: listenerName1,
|
||||
ApiListener: &v3listenerpb.ApiListener{
|
||||
ApiListener: xdstestutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
|
||||
RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
|
||||
}),
|
||||
},
|
||||
})},
|
||||
},
|
||||
wantUpdates: map[string]dataAndErrTuple{
|
||||
listenerName1: {
|
||||
Err: cmpopts.AnyError,
|
||||
},
|
||||
},
|
||||
wantMD: xdsresource.UpdateMetadata{
|
||||
Status: xdsresource.ServiceStatusNACKed,
|
||||
Version: "0",
|
||||
ErrState: &xdsresource.UpdateErrorMetadata{
|
||||
Version: "0",
|
||||
Err: cmpopts.AnyError,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "two bad resources",
|
||||
resourceNamesToRequest: []string{listenerName1, listenerName2},
|
||||
managementServerResponse: &v3discoverypb.DiscoveryResponse{
|
||||
VersionInfo: "0",
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
Resources: []*anypb.Any{
|
||||
badlyMarshaledResource,
|
||||
xdstestutils.MarshalAny(t, &v3listenerpb.Listener{
|
||||
Name: listenerName2,
|
||||
ApiListener: &v3listenerpb.ApiListener{
|
||||
ApiListener: xdstestutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
|
||||
RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
|
||||
}),
|
||||
},
|
||||
}),
|
||||
},
|
||||
},
|
||||
wantUpdates: map[string]dataAndErrTuple{
|
||||
listenerName2: {
|
||||
Err: cmpopts.AnyError,
|
||||
},
|
||||
},
|
||||
wantMD: xdsresource.UpdateMetadata{
|
||||
Status: xdsresource.ServiceStatusNACKed,
|
||||
Version: "0",
|
||||
ErrState: &xdsresource.UpdateErrorMetadata{
|
||||
Version: "0",
|
||||
Err: cmpopts.AnyError,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "one good resource",
|
||||
resourceNamesToRequest: []string{listenerName1},
|
||||
managementServerResponse: &v3discoverypb.DiscoveryResponse{
|
||||
VersionInfo: "0",
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
Resources: []*anypb.Any{listener1},
|
||||
},
|
||||
wantUpdates: map[string]dataAndErrTuple{
|
||||
listenerName1: {
|
||||
Resource: &listenerResourceData{Resource: listenerUpdate{
|
||||
RouteConfigName: routeName,
|
||||
Raw: listener1.GetValue(),
|
||||
}},
|
||||
},
|
||||
},
|
||||
wantMD: xdsresource.UpdateMetadata{
|
||||
Status: xdsresource.ServiceStatusACKed,
|
||||
Version: "0",
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "one good and one bad - deserialization failure",
|
||||
resourceNamesToRequest: []string{listenerName1, listenerName2},
|
||||
managementServerResponse: &v3discoverypb.DiscoveryResponse{
|
||||
VersionInfo: "0",
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
Resources: []*anypb.Any{
|
||||
badlyMarshaledResource,
|
||||
listener2,
|
||||
},
|
||||
},
|
||||
wantUpdates: map[string]dataAndErrTuple{
|
||||
listenerName2: {
|
||||
Resource: &listenerResourceData{Resource: listenerUpdate{
|
||||
RouteConfigName: routeName,
|
||||
Raw: listener2.GetValue(),
|
||||
}},
|
||||
},
|
||||
},
|
||||
wantMD: xdsresource.UpdateMetadata{
|
||||
Status: xdsresource.ServiceStatusNACKed,
|
||||
Version: "0",
|
||||
ErrState: &xdsresource.UpdateErrorMetadata{
|
||||
Version: "0",
|
||||
Err: cmpopts.AnyError,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "one good and one bad - validation failure",
|
||||
resourceNamesToRequest: []string{listenerName1, listenerName2},
|
||||
managementServerResponse: &v3discoverypb.DiscoveryResponse{
|
||||
VersionInfo: "0",
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
Resources: []*anypb.Any{
|
||||
xdstestutils.MarshalAny(t, &v3listenerpb.Listener{
|
||||
Name: listenerName1,
|
||||
ApiListener: &v3listenerpb.ApiListener{
|
||||
ApiListener: xdstestutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
|
||||
RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
|
||||
}),
|
||||
},
|
||||
}),
|
||||
listener2,
|
||||
},
|
||||
},
|
||||
wantUpdates: map[string]dataAndErrTuple{
|
||||
listenerName1: {Err: cmpopts.AnyError},
|
||||
listenerName2: {
|
||||
Resource: &listenerResourceData{Resource: listenerUpdate{
|
||||
RouteConfigName: routeName,
|
||||
Raw: listener2.GetValue(),
|
||||
}},
|
||||
},
|
||||
},
|
||||
wantMD: xdsresource.UpdateMetadata{
|
||||
Status: xdsresource.ServiceStatusNACKed,
|
||||
Version: "0",
|
||||
ErrState: &xdsresource.UpdateErrorMetadata{
|
||||
Version: "0",
|
||||
Err: cmpopts.AnyError,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "two good resources",
|
||||
resourceNamesToRequest: []string{listenerName1, listenerName2},
|
||||
managementServerResponse: &v3discoverypb.DiscoveryResponse{
|
||||
VersionInfo: "0",
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
Resources: []*anypb.Any{listener1, listener2},
|
||||
},
|
||||
wantUpdates: map[string]dataAndErrTuple{
|
||||
listenerName1: {
|
||||
Resource: &listenerResourceData{Resource: listenerUpdate{
|
||||
RouteConfigName: routeName,
|
||||
Raw: listener1.GetValue(),
|
||||
}},
|
||||
},
|
||||
listenerName2: {
|
||||
Resource: &listenerResourceData{Resource: listenerUpdate{
|
||||
RouteConfigName: routeName,
|
||||
Raw: listener2.GetValue(),
|
||||
}},
|
||||
},
|
||||
},
|
||||
wantMD: xdsresource.UpdateMetadata{
|
||||
Status: xdsresource.ServiceStatusACKed,
|
||||
Version: "0",
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "two resources when we requested one",
|
||||
resourceNamesToRequest: []string{listenerName1},
|
||||
managementServerResponse: &v3discoverypb.DiscoveryResponse{
|
||||
VersionInfo: "0",
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
Resources: []*anypb.Any{listener1, listener2},
|
||||
},
|
||||
wantUpdates: map[string]dataAndErrTuple{
|
||||
listenerName1: {
|
||||
Resource: &listenerResourceData{Resource: listenerUpdate{
|
||||
RouteConfigName: routeName,
|
||||
Raw: listener1.GetValue(),
|
||||
}},
|
||||
},
|
||||
listenerName2: {
|
||||
Resource: &listenerResourceData{Resource: listenerUpdate{
|
||||
RouteConfigName: routeName,
|
||||
Raw: listener2.GetValue(),
|
||||
}},
|
||||
},
|
||||
},
|
||||
wantMD: xdsresource.UpdateMetadata{
|
||||
Status: xdsresource.ServiceStatusACKed,
|
||||
Version: "0",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Start a fake xDS management server and configure the response it
|
||||
// would send to its client.
|
||||
mgmtServer, cleanup, err := fakeserver.StartServer(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to start fake xDS server: %v", err)
|
||||
}
|
||||
defer cleanup()
|
||||
t.Logf("Started xDS management server on %s", mgmtServer.Address)
|
||||
mgmtServer.XDSResponseChan <- &fakeserver.Response{Resp: test.managementServerResponse}
|
||||
|
||||
// Create an xdsChannel for the test with a long watch expiry timer
|
||||
// to ensure that watches don't expire for the duration of the test.
|
||||
nodeID := uuid.New().String()
|
||||
xc := xdsChannelForTest(t, mgmtServer.Address, nodeID, 2*defaultTestTimeout)
|
||||
defer xc.close()
|
||||
|
||||
// Subscribe to the resources specified in the test table.
|
||||
for _, name := range test.resourceNamesToRequest {
|
||||
xc.subscribe(listenerType, name)
|
||||
}
|
||||
|
||||
// Wait for an update callback on the event handler and verify the
|
||||
// contents of the update and the metadata.
|
||||
verifyUpdateAndMetadata(ctx, t, xc.eventHandler.(*testEventHandler), test.wantUpdates, test.wantMD)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Tests that the xdsChannel correctly handles the expiry of a watch for a
|
||||
// resource by ensuring that the watch expiry callback is invoked on the event
|
||||
// handler with the expected resource type and name.
|
||||
func (s) TestChannel_ADS_HandleResponseWatchExpiry(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Start an xDS management server, but do not configure any resources on it.
|
||||
// This will result in the watch for a resource to timeout.
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
|
||||
|
||||
// Create an xdsChannel for the test with a short watch expiry timer to
|
||||
// ensure that the test does not run very long, as it needs to wait for the
|
||||
// watch to expire.
|
||||
nodeID := uuid.New().String()
|
||||
xc := xdsChannelForTest(t, mgmtServer.Address, nodeID, 2*defaultTestShortTimeout)
|
||||
defer xc.close()
|
||||
|
||||
// Subscribe to a listener resource.
|
||||
const listenerName = "listener-name"
|
||||
xc.subscribe(listenerType, listenerName)
|
||||
|
||||
// Wait for the watch expiry callback on the authority to be invoked and
|
||||
// verify that the watch expired for the expected resource name and type.
|
||||
eventHandler := xc.eventHandler.(*testEventHandler)
|
||||
gotTyp, gotName, err := eventHandler.waitForResourceDoesNotExist(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Timeout when waiting for the watch expiry callback to be invoked on the xDS client")
|
||||
}
|
||||
|
||||
if gotTyp != listenerType {
|
||||
t.Fatalf("Got type %v, want %v", gotTyp, listenerType)
|
||||
}
|
||||
if gotName != listenerName {
|
||||
t.Fatalf("Got name %v, want %v", gotName, listenerName)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests that the xdsChannel correctly handles stream failures by ensuring that
|
||||
// the stream failure callback is invoked on the event handler.
|
||||
func (s) TestChannel_ADS_StreamFailure(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20000*defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Start an xDS management server with a restartable listener to simulate
|
||||
// connection failures.
|
||||
l, err := net.Listen("tcp", "localhost:0")
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen() failed: %v", err)
|
||||
}
|
||||
lis := xdstestutils.NewRestartableListener(l)
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lis})
|
||||
|
||||
// Configure a listener resource on the management server.
|
||||
const listenerResourceName = "test-listener-resource"
|
||||
const routeConfigurationName = "test-route-configuration-resource"
|
||||
nodeID := uuid.New().String()
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
|
||||
// Create an xdsChannel for the test with a long watch expiry timer
|
||||
// to ensure that watches don't expire for the duration of the test.
|
||||
xc := xdsChannelForTest(t, mgmtServer.Address, nodeID, 2000*defaultTestTimeout)
|
||||
defer xc.close()
|
||||
|
||||
// Subscribe to the resource created above.
|
||||
xc.subscribe(listenerType, listenerResourceName)
|
||||
|
||||
// Wait for an update callback on the event handler and verify the
|
||||
// contents of the update and the metadata.
|
||||
hcm := xdstestutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
|
||||
RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{Rds: &v3httppb.Rds{
|
||||
ConfigSource: &v3corepb.ConfigSource{
|
||||
ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
|
||||
},
|
||||
RouteConfigName: routeConfigurationName,
|
||||
}},
|
||||
})
|
||||
listenerResource, err := anypb.New(&v3listenerpb.Listener{
|
||||
Name: listenerResourceName,
|
||||
ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create listener resource: %v", err)
|
||||
}
|
||||
|
||||
wantUpdates := map[string]dataAndErrTuple{
|
||||
listenerResourceName: {
|
||||
Resource: &listenerResourceData{
|
||||
Resource: listenerUpdate{
|
||||
RouteConfigName: routeConfigurationName,
|
||||
Raw: listenerResource.GetValue(),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
wantMD := xdsresource.UpdateMetadata{
|
||||
Status: xdsresource.ServiceStatusACKed,
|
||||
Version: "1",
|
||||
}
|
||||
|
||||
eventHandler := xc.eventHandler.(*testEventHandler)
|
||||
verifyUpdateAndMetadata(ctx, t, eventHandler, wantUpdates, wantMD)
|
||||
|
||||
lis.Stop()
|
||||
if err := eventHandler.waitForStreamFailure(ctx); err != nil {
|
||||
t.Fatalf("Timeout when waiting for the stream failure callback to be invoked on the xDS client: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests the behavior of the xdsChannel when a resource is unsubscribed.
|
||||
// Verifies that when a previously subscribed resource is unsubscribed, a
|
||||
// request is sent without the previously subscribed resource name.
|
||||
func (s) TestChannel_ADS_ResourceUnsubscribe(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Start an xDS management server that uses a channel to inform the test
|
||||
// about the specific LDS resource names being requested.
|
||||
ldsResourcesCh := make(chan []string, 1)
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
|
||||
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
|
||||
t.Logf("Received request for resources: %v of type %s", req.GetResourceNames(), req.GetTypeUrl())
|
||||
|
||||
if req.TypeUrl != xdsresource.V3ListenerURL {
|
||||
return fmt.Errorf("unexpected resource type URL: %q", req.TypeUrl)
|
||||
}
|
||||
|
||||
// Make the most recently requested names available to the test.
|
||||
ldsResourcesCh <- req.GetResourceNames()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
// Configure two listener resources on the management server.
|
||||
const listenerResourceName1 = "test-listener-resource-1"
|
||||
const routeConfigurationName1 = "test-route-configuration-resource-1"
|
||||
const listenerResourceName2 = "test-listener-resource-2"
|
||||
const routeConfigurationName2 = "test-route-configuration-resource-2"
|
||||
nodeID := uuid.New().String()
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{
|
||||
e2e.DefaultClientListener(listenerResourceName1, routeConfigurationName1),
|
||||
e2e.DefaultClientListener(listenerResourceName2, routeConfigurationName2),
|
||||
},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
|
||||
// Create an xdsChannel for the test with a long watch expiry timer
|
||||
// to ensure that watches don't expire for the duration of the test.
|
||||
xc := xdsChannelForTest(t, mgmtServer.Address, nodeID, 2*defaultTestTimeout)
|
||||
defer xc.close()
|
||||
|
||||
// Subscribe to the resources created above and verify that a request is
|
||||
// sent for the same.
|
||||
xc.subscribe(listenerType, listenerResourceName1)
|
||||
xc.subscribe(listenerType, listenerResourceName2)
|
||||
if err := waitForResourceNames(ctx, ldsResourcesCh, []string{listenerResourceName1, listenerResourceName2}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Wait for the above resources to be ACKed.
|
||||
if err := waitForResourceNames(ctx, ldsResourcesCh, []string{listenerResourceName1, listenerResourceName2}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Unsubscribe to one of the resources created above, and ensure that the
|
||||
// other resource is still being requested.
|
||||
xc.unsubscribe(listenerType, listenerResourceName1)
|
||||
if err := waitForResourceNames(ctx, ldsResourcesCh, []string{listenerResourceName2}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Since the version on the management server for the above resource is not
|
||||
// changed, we will not receive an update from it for the one resource that
|
||||
// we are still requesting.
|
||||
|
||||
// Unsubscribe to the remaining resource, and ensure that no more resources
|
||||
// are being requested.
|
||||
xc.unsubscribe(listenerType, listenerResourceName2)
|
||||
if err := waitForResourceNames(ctx, ldsResourcesCh, []string{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// waitForResourceNames waits for the wantNames to be received on namesCh.
|
||||
// Returns a non-nil error if the context expires before that.
|
||||
func waitForResourceNames(ctx context.Context, namesCh chan []string, wantNames []string) error {
|
||||
var lastRequestedNames []string
|
||||
for ; ; <-time.After(defaultTestShortTimeout) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("timeout waiting for resources %v to be requested from the management server. Last requested resources: %v", wantNames, lastRequestedNames)
|
||||
case gotNames := <-namesCh:
|
||||
if cmp.Equal(gotNames, wantNames, cmpopts.EquateEmpty(), cmpopts.SortSlices(func(s1, s2 string) bool { return s1 < s2 })) {
|
||||
return nil
|
||||
}
|
||||
lastRequestedNames = gotNames
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// newTestEventHandler creates a new testEventHandler instance with the
|
||||
// necessary channels for testing the xdsChannel.
|
||||
func newTestEventHandler() *testEventHandler {
|
||||
return &testEventHandler{
|
||||
typeCh: make(chan ResourceType, 1),
|
||||
updateCh: make(chan map[string]dataAndErrTuple, 1),
|
||||
mdCh: make(chan xdsresource.UpdateMetadata, 1),
|
||||
nameCh: make(chan string, 1),
|
||||
connErrCh: make(chan error, 1),
|
||||
}
|
||||
}
|
||||
|
||||
// testEventHandler is a struct that implements the xdsChannelEventhandler
|
||||
// interface. It is used to receive events from an xdsChannel, and has multiple
|
||||
// channels on which it makes these events available to the test.
|
||||
type testEventHandler struct {
|
||||
typeCh chan ResourceType // Resource type of an update or resource-does-not-exist error.
|
||||
updateCh chan map[string]dataAndErrTuple // Resource updates.
|
||||
mdCh chan xdsresource.UpdateMetadata // Metadata from an update.
|
||||
nameCh chan string // Name of the non-existent resource.
|
||||
connErrCh chan error // Connectivity error.
|
||||
|
||||
}
|
||||
|
||||
func (ta *testEventHandler) adsStreamFailure(err error) {
|
||||
ta.connErrCh <- err
|
||||
}
|
||||
|
||||
func (ta *testEventHandler) waitForStreamFailure(ctx context.Context) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ta.connErrCh:
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ta *testEventHandler) adsResourceUpdate(typ ResourceType, updates map[string]dataAndErrTuple, md xdsresource.UpdateMetadata, onDone func()) {
|
||||
ta.typeCh <- typ
|
||||
ta.updateCh <- updates
|
||||
ta.mdCh <- md
|
||||
onDone()
|
||||
}
|
||||
|
||||
// waitForUpdate waits for the next resource update event from the xdsChannel.
|
||||
// It returns the resource type, the resource updates, and the update metadata.
|
||||
// If the context is canceled, it returns an error.
|
||||
func (ta *testEventHandler) waitForUpdate(ctx context.Context) (ResourceType, map[string]dataAndErrTuple, xdsresource.UpdateMetadata, error) {
|
||||
var typ ResourceType
|
||||
var updates map[string]dataAndErrTuple
|
||||
var md xdsresource.UpdateMetadata
|
||||
|
||||
select {
|
||||
case typ = <-ta.typeCh:
|
||||
case <-ctx.Done():
|
||||
return ResourceType{}, nil, xdsresource.UpdateMetadata{}, ctx.Err()
|
||||
}
|
||||
|
||||
select {
|
||||
case updates = <-ta.updateCh:
|
||||
case <-ctx.Done():
|
||||
return ResourceType{}, nil, xdsresource.UpdateMetadata{}, ctx.Err()
|
||||
}
|
||||
|
||||
select {
|
||||
case md = <-ta.mdCh:
|
||||
case <-ctx.Done():
|
||||
return ResourceType{}, nil, xdsresource.UpdateMetadata{}, ctx.Err()
|
||||
}
|
||||
return typ, updates, md, nil
|
||||
}
|
||||
|
||||
func (ta *testEventHandler) adsResourceDoesNotExist(typ ResourceType, name string) {
|
||||
ta.typeCh <- typ
|
||||
ta.nameCh <- name
|
||||
}
|
||||
|
||||
// waitForResourceDoesNotExist waits for the next resource-does-not-exist event
|
||||
// from the xdsChannel. It returns the resource type and the resource name. If
|
||||
// the context is canceled, it returns an error.
|
||||
func (ta *testEventHandler) waitForResourceDoesNotExist(ctx context.Context) (ResourceType, string, error) {
|
||||
var typ ResourceType
|
||||
var name string
|
||||
|
||||
select {
|
||||
case typ = <-ta.typeCh:
|
||||
case <-ctx.Done():
|
||||
return ResourceType{}, "", ctx.Err()
|
||||
}
|
||||
|
||||
select {
|
||||
case name = <-ta.nameCh:
|
||||
case <-ctx.Done():
|
||||
return ResourceType{}, "", ctx.Err()
|
||||
}
|
||||
return typ, name, nil
|
||||
}
|
||||
|
||||
type panicDecoder struct{}
|
||||
|
||||
func (panicDecoder) Decode(*AnyProto, DecodeOptions) (*DecodeResult, error) {
|
||||
panic("simulate panic")
|
||||
}
|
||||
|
||||
// TestDecodeResponse_PanicRecoveryEnabled tests the panic recovery mechanism
|
||||
// in decodeResponse. It verifies that if XDSRecoverPanicInResourceParsing
|
||||
// env variable is enabled, panics during unmarshaling are caught and returned
|
||||
// as errors.
|
||||
func (s) TestDecodeResponse_PanicRecoveryEnabled(t *testing.T) {
|
||||
rType := &ResourceType{
|
||||
TypeName: "resourceType",
|
||||
Decoder: panicDecoder{},
|
||||
}
|
||||
resp := response{resources: []*anypb.Any{{Value: []byte("test")}}}
|
||||
wantErr := "recovered from panic during resource parsing"
|
||||
|
||||
xc := &xdsChannel{}
|
||||
if _, _, err := xc.decodeResponse(rType, resp); err == nil || !strings.Contains(err.Error(), wantErr) {
|
||||
t.Fatalf("decodeResponse() failed with err: %v, want %q", err, wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecodeResponse_PanicRecoveryDisabled tests the panic recovery mechanism
|
||||
// in decodeResponse. It verifies that when XDSRecoverPanicInResourceParsing
|
||||
// env variable is disabled, panics during unmarshaling propagate.
|
||||
func (s) TestDecodeResponse_PanicRecoveryDisabled(t *testing.T) {
|
||||
testutils.SetEnvConfig(t, &envconfig.XDSRecoverPanicInResourceParsing, false)
|
||||
rType := &ResourceType{
|
||||
TypeName: "resourceType",
|
||||
Decoder: panicDecoder{},
|
||||
}
|
||||
resp := response{resources: []*anypb.Any{{Value: []byte("test")}}}
|
||||
wantErr := "simulate panic"
|
||||
xc := &xdsChannel{}
|
||||
|
||||
defer func() {
|
||||
if r := recover(); r == nil || !strings.Contains(fmt.Sprint(r), wantErr) {
|
||||
t.Fatalf("Expected panic in decodeResponse, got: %v, want: %q", r, wantErr)
|
||||
}
|
||||
}()
|
||||
xc.decodeResponse(rType, resp)
|
||||
}
|
||||
109
internal/xds/clients/xdsclient/clientimpl_watchers.go
Normal file
109
internal/xds/clients/xdsclient/clientimpl_watchers.go
Normal file
@@ -0,0 +1,109 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
)
|
||||
|
||||
// wrappingWatcher is a wrapper around an xdsresource.ResourceWatcher that adds
|
||||
// the node ID to the error messages reported to the watcher.
|
||||
type wrappingWatcher struct {
|
||||
ResourceWatcher
|
||||
nodeID string
|
||||
}
|
||||
|
||||
func (w *wrappingWatcher) AmbientError(err error, done func()) {
|
||||
w.ResourceWatcher.AmbientError(fmt.Errorf("[xDS node id: %v]: %w", w.nodeID, err), done)
|
||||
}
|
||||
|
||||
func (w *wrappingWatcher) ResourceError(err error, done func()) {
|
||||
w.ResourceWatcher.ResourceError(fmt.Errorf("[xDS node id: %v]: %w", w.nodeID, err), done)
|
||||
}
|
||||
|
||||
// WatchResource starts watching the specified resource.
|
||||
//
|
||||
// The watch fails to start if:
|
||||
// - There is no ResourceType implementation for the given typeURL in the
|
||||
// ResourceTypes field of the Config struct used to create the XDSClient.
|
||||
// - The provided resourceName contains an authority that is not present in the
|
||||
// Authorities field.
|
||||
//
|
||||
// The returned function cancels the watch and prevents future calls to the
|
||||
// watcher.
|
||||
func (c *XDSClient) WatchResource(typeURL, resourceName string, watcher ResourceWatcher) (cancel func()) {
|
||||
// Return early if the client is already closed.
|
||||
if c.done.HasFired() {
|
||||
logger.Warningf("Watch registered for type %q, but client is closed", typeURL)
|
||||
return func() {}
|
||||
}
|
||||
|
||||
watcher = &wrappingWatcher{
|
||||
ResourceWatcher: watcher,
|
||||
nodeID: c.config.Node.ID,
|
||||
}
|
||||
|
||||
rType, ok := c.config.ResourceTypes[typeURL]
|
||||
if !ok {
|
||||
logger.Warningf("ResourceType implementation for resource type url %q is not found", rType.TypeURL)
|
||||
c.serializer.TrySchedule(func(context.Context) {
|
||||
watcher.ResourceError(fmt.Errorf("no ResourceType implementation found for typeURL %q", rType.TypeURL), func() {})
|
||||
})
|
||||
return func() {}
|
||||
}
|
||||
|
||||
n := xdsresource.ParseName(resourceName)
|
||||
a := c.getAuthorityForResource(n)
|
||||
if a == nil {
|
||||
logger.Warningf("Watch registered for name %q of type %q, authority %q is not found", rType.TypeName, resourceName, n.Authority)
|
||||
c.serializer.TrySchedule(func(context.Context) {
|
||||
watcher.ResourceError(fmt.Errorf("authority %q not found in the config for resource %q", n.Authority, resourceName), func() {})
|
||||
})
|
||||
return func() {}
|
||||
}
|
||||
// The watchResource method on the authority is invoked with n.String()
|
||||
// instead of resourceName because n.String() canonicalizes the given name.
|
||||
// So, two resource names which don't differ in the query string, but only
|
||||
// differ in the order of context params will result in the same resource
|
||||
// being watched by the authority.
|
||||
return a.watchResource(rType, n.String(), watcher)
|
||||
}
|
||||
|
||||
// Gets the authority for the given resource name.
|
||||
//
|
||||
// See examples in this section of the gRFC:
|
||||
// https://github.com/grpc/proposal/blob/master/A47-xds-federation.md#bootstrap-config-changes
|
||||
func (c *XDSClient) getAuthorityForResource(name *xdsresource.Name) *authority {
|
||||
// For new-style resource names, always lookup the authorities map. If the
|
||||
// name does not specify an authority, we will end up looking for an entry
|
||||
// in the map with the empty string as the key.
|
||||
if name.Scheme == xdsresource.FederationScheme {
|
||||
return c.authorities[name.Authority]
|
||||
}
|
||||
|
||||
// For old-style resource names, we use the top-level authority if the name
|
||||
// does not specify an authority.
|
||||
if name.Authority == "" {
|
||||
return c.topLevelAuthority
|
||||
}
|
||||
return c.authorities[name.Authority]
|
||||
}
|
||||
233
internal/xds/clients/xdsclient/helpers_test.go
Normal file
233
internal/xds/clients/xdsclient/helpers_test.go
Normal file
@@ -0,0 +1,233 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc/internal/grpctest"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/pretty"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
|
||||
v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
|
||||
)
|
||||
|
||||
type s struct {
|
||||
grpctest.Tester
|
||||
}
|
||||
|
||||
func Test(t *testing.T) {
|
||||
grpctest.RunSubTests(t, s{})
|
||||
}
|
||||
|
||||
const (
|
||||
defaultTestTimeout = 5 * time.Second
|
||||
defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen.
|
||||
// listenerResourceTypeName represents the transport agnostic name for the
|
||||
// listener resource.
|
||||
listenerResourceTypeName = "ListenerResource"
|
||||
)
|
||||
|
||||
var (
|
||||
// Singleton instantiation of the resource type implementation.
|
||||
listenerType = ResourceType{
|
||||
TypeURL: xdsresource.V3ListenerURL,
|
||||
TypeName: listenerResourceTypeName,
|
||||
AllResourcesRequiredInSotW: true,
|
||||
Decoder: listenerDecoder{},
|
||||
}
|
||||
)
|
||||
|
||||
func unmarshalListenerResource(rProto *anypb.Any) (string, listenerUpdate, error) {
|
||||
rProto, err := xdsresource.UnwrapResource(rProto)
|
||||
if err != nil {
|
||||
return "", listenerUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err)
|
||||
}
|
||||
if !xdsresource.IsListenerResource(rProto.GetTypeUrl()) {
|
||||
return "", listenerUpdate{}, fmt.Errorf("unexpected listener resource type: %q", rProto.GetTypeUrl())
|
||||
}
|
||||
lis := &v3listenerpb.Listener{}
|
||||
if err := proto.Unmarshal(rProto.GetValue(), lis); err != nil {
|
||||
return "", listenerUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
|
||||
}
|
||||
|
||||
lu, err := processListener(lis)
|
||||
if err != nil {
|
||||
return lis.GetName(), listenerUpdate{}, err
|
||||
}
|
||||
lu.Raw = rProto.GetValue()
|
||||
return lis.GetName(), *lu, nil
|
||||
}
|
||||
|
||||
func processListener(lis *v3listenerpb.Listener) (*listenerUpdate, error) {
|
||||
if lis.GetApiListener() != nil {
|
||||
return processClientSideListener(lis)
|
||||
}
|
||||
return processServerSideListener(lis)
|
||||
}
|
||||
|
||||
// processClientSideListener checks if the provided Listener proto meets
|
||||
// the expected criteria. If so, it returns a non-empty routeConfigName.
|
||||
func processClientSideListener(lis *v3listenerpb.Listener) (*listenerUpdate, error) {
|
||||
update := &listenerUpdate{}
|
||||
|
||||
apiLisAny := lis.GetApiListener().GetApiListener()
|
||||
if !xdsresource.IsHTTPConnManagerResource(apiLisAny.GetTypeUrl()) {
|
||||
return nil, fmt.Errorf("unexpected http connection manager resource type: %q", apiLisAny.GetTypeUrl())
|
||||
}
|
||||
apiLis := &v3httppb.HttpConnectionManager{}
|
||||
if err := proto.Unmarshal(apiLisAny.GetValue(), apiLis); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal api_listener: %v", err)
|
||||
}
|
||||
|
||||
switch apiLis.RouteSpecifier.(type) {
|
||||
case *v3httppb.HttpConnectionManager_Rds:
|
||||
if configsource := apiLis.GetRds().GetConfigSource(); configsource.GetAds() == nil && configsource.GetSelf() == nil {
|
||||
return nil, fmt.Errorf("LDS's RDS configSource is not ADS or Self: %+v", lis)
|
||||
}
|
||||
name := apiLis.GetRds().GetRouteConfigName()
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("empty route_config_name: %+v", lis)
|
||||
}
|
||||
update.RouteConfigName = name
|
||||
case *v3httppb.HttpConnectionManager_RouteConfig:
|
||||
routeU := apiLis.GetRouteConfig()
|
||||
if routeU == nil {
|
||||
return nil, fmt.Errorf("empty inline RDS resp:: %+v", lis)
|
||||
}
|
||||
if routeU.Name == "" {
|
||||
return nil, fmt.Errorf("empty route_config_name in inline RDS resp: %+v", lis)
|
||||
}
|
||||
update.RouteConfigName = routeU.Name
|
||||
case nil:
|
||||
return nil, fmt.Errorf("no RouteSpecifier: %+v", apiLis)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported type %T for RouteSpecifier", apiLis.RouteSpecifier)
|
||||
}
|
||||
|
||||
return update, nil
|
||||
}
|
||||
|
||||
func processServerSideListener(lis *v3listenerpb.Listener) (*listenerUpdate, error) {
|
||||
if n := len(lis.ListenerFilters); n != 0 {
|
||||
return nil, fmt.Errorf("unsupported field 'listener_filters' contains %d entries", n)
|
||||
}
|
||||
if lis.GetUseOriginalDst().GetValue() {
|
||||
return nil, errors.New("unsupported field 'use_original_dst' is present and set to true")
|
||||
}
|
||||
addr := lis.GetAddress()
|
||||
if addr == nil {
|
||||
return nil, fmt.Errorf("no address field in LDS response: %+v", lis)
|
||||
}
|
||||
sockAddr := addr.GetSocketAddress()
|
||||
if sockAddr == nil {
|
||||
return nil, fmt.Errorf("no socket_address field in LDS response: %+v", lis)
|
||||
}
|
||||
lu := &listenerUpdate{
|
||||
InboundListenerCfg: &inboundListenerConfig{
|
||||
Address: sockAddr.GetAddress(),
|
||||
Port: strconv.Itoa(int(sockAddr.GetPortValue())),
|
||||
},
|
||||
}
|
||||
|
||||
return lu, nil
|
||||
}
|
||||
|
||||
type listenerDecoder struct{}
|
||||
|
||||
// Decode deserializes and validates an xDS resource serialized inside the
|
||||
// provided `Any` proto, as received from the xDS management server.
|
||||
func (listenerDecoder) Decode(resource *AnyProto, _ DecodeOptions) (*DecodeResult, error) {
|
||||
name, listener, err := unmarshalListenerResource(resource.ToAny())
|
||||
switch {
|
||||
case name == "":
|
||||
// Name is unset only when protobuf deserialization fails.
|
||||
return nil, err
|
||||
case err != nil:
|
||||
// Protobuf deserialization succeeded, but resource validation failed.
|
||||
return &DecodeResult{Name: name, Resource: &listenerResourceData{Resource: listenerUpdate{}}}, err
|
||||
}
|
||||
|
||||
return &DecodeResult{Name: name, Resource: &listenerResourceData{Resource: listener}}, nil
|
||||
|
||||
}
|
||||
|
||||
// listenerResourceData wraps the configuration of a Listener resource as
|
||||
// received from the management server.
|
||||
//
|
||||
// Implements the ResourceData interface.
|
||||
type listenerResourceData struct {
|
||||
ResourceData
|
||||
|
||||
Resource listenerUpdate
|
||||
}
|
||||
|
||||
// Equal returns true if other is equal to l.
|
||||
func (l *listenerResourceData) Equal(other ResourceData) bool {
|
||||
if l == nil && other == nil {
|
||||
return true
|
||||
}
|
||||
if (l == nil) != (other == nil) {
|
||||
return false
|
||||
}
|
||||
return bytes.Equal(l.Resource.Raw, other.Bytes())
|
||||
}
|
||||
|
||||
// ToJSON returns a JSON string representation of the resource data.
|
||||
func (l *listenerResourceData) ToJSON() string {
|
||||
return pretty.ToJSON(l.Resource)
|
||||
}
|
||||
|
||||
// Bytes returns the underlying raw protobuf form of the listener resource.
|
||||
func (l *listenerResourceData) Bytes() []byte {
|
||||
return l.Resource.Raw
|
||||
}
|
||||
|
||||
// ListenerUpdate contains information received in an LDS response, which is of
|
||||
// interest to the registered LDS watcher.
|
||||
type listenerUpdate struct {
|
||||
// RouteConfigName is the route configuration name corresponding to the
|
||||
// target which is being watched through LDS.
|
||||
RouteConfigName string
|
||||
|
||||
// InboundListenerCfg contains inbound listener configuration.
|
||||
InboundListenerCfg *inboundListenerConfig
|
||||
|
||||
// Raw is the resource from the xds response.
|
||||
Raw []byte
|
||||
}
|
||||
|
||||
// InboundListenerConfig contains information about the inbound listener, i.e
|
||||
// the server-side listener.
|
||||
type inboundListenerConfig struct {
|
||||
// Address is the local address on which the inbound listener is expected to
|
||||
// accept incoming connections.
|
||||
Address string
|
||||
// Port is the local port on which the inbound listener is expected to
|
||||
// accept incoming connections.
|
||||
Port string
|
||||
}
|
||||
32
internal/xds/clients/xdsclient/internal/internal.go
Normal file
32
internal/xds/clients/xdsclient/internal/internal.go
Normal file
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Package internal contains functionality internal to the xdsclient package.
|
||||
package internal
|
||||
|
||||
import "time"
|
||||
|
||||
var (
|
||||
// StreamBackoff is the stream backoff for xDS client. It can be overridden
|
||||
// by tests to change the default backoff strategy.
|
||||
StreamBackoff func(int) time.Duration
|
||||
|
||||
// ResourceWatchStateForTesting gets the watch state for the resource
|
||||
// identified by the given resource type and resource name. Returns a
|
||||
// non-nil error if there is no such resource being watched.
|
||||
ResourceWatchStateForTesting any // func(*xdsclient.XDSClient, xdsclient.ResourceType, string) error
|
||||
)
|
||||
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2021 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package xdsresource
|
||||
|
||||
import "time"
|
||||
|
||||
// WatchState is a enum that describes the watch state of a particular
|
||||
// resource.
|
||||
type WatchState int
|
||||
|
||||
const (
|
||||
// ResourceWatchStateStarted is the state where a watch for a resource was
|
||||
// started, but a request asking for that resource is yet to be sent to the
|
||||
// management server.
|
||||
ResourceWatchStateStarted WatchState = iota
|
||||
// ResourceWatchStateRequested is the state when a request has been sent for
|
||||
// the resource being watched.
|
||||
ResourceWatchStateRequested
|
||||
// ResourceWatchStateReceived is the state when a response has been received
|
||||
// for the resource being watched.
|
||||
ResourceWatchStateReceived
|
||||
// ResourceWatchStateTimeout is the state when the watch timer associated
|
||||
// with the resource expired because no response was received.
|
||||
ResourceWatchStateTimeout
|
||||
)
|
||||
|
||||
// ResourceWatchState is the state corresponding to a resource being watched.
|
||||
type ResourceWatchState struct {
|
||||
State WatchState // Watch state of the resource.
|
||||
ExpiryTimer *time.Timer // Timer for the expiry of the watch.
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2021 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package xdsresource
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// ErrorType is the type of the error that the watcher will receive from the xds
|
||||
// client.
|
||||
type ErrorType int
|
||||
|
||||
const (
|
||||
// ErrorTypeUnknown indicates the error doesn't have a specific type. It is
|
||||
// the default value, and is returned if the error is not an xds error.
|
||||
ErrorTypeUnknown ErrorType = iota
|
||||
// ErrorTypeConnection indicates a connection error from the gRPC client.
|
||||
ErrorTypeConnection
|
||||
// ErrorTypeResourceNotFound indicates a resource is not found from the xds
|
||||
// response. It's typically returned if the resource is removed in the xds
|
||||
// server.
|
||||
ErrorTypeResourceNotFound
|
||||
// ErrorTypeResourceTypeUnsupported indicates the receipt of a message from
|
||||
// the management server with resources of an unsupported resource type.
|
||||
ErrorTypeResourceTypeUnsupported
|
||||
// ErrTypeStreamFailedAfterRecv indicates an ADS stream error, after
|
||||
// successful receipt of at least one message from the server.
|
||||
ErrTypeStreamFailedAfterRecv
|
||||
// ErrorTypeNACKed indicates that configuration provided by the xDS management
|
||||
// server was NACKed.
|
||||
ErrorTypeNACKed
|
||||
)
|
||||
|
||||
type xdsClientError struct {
|
||||
t ErrorType
|
||||
desc string
|
||||
}
|
||||
|
||||
func (e *xdsClientError) Error() string {
|
||||
return e.desc
|
||||
}
|
||||
|
||||
// NewErrorf creates an xDS client error. The callbacks are called with this
|
||||
// error, to pass additional information about the error.
|
||||
func NewErrorf(t ErrorType, format string, args ...any) error {
|
||||
return &xdsClientError{t: t, desc: fmt.Sprintf(format, args...)}
|
||||
}
|
||||
|
||||
// NewError creates an xDS client error. The callbacks are called with this
|
||||
// error, to pass additional information about the error.
|
||||
func NewError(t ErrorType, message string) error {
|
||||
return NewErrorf(t, "%s", message)
|
||||
}
|
||||
|
||||
// ErrType returns the error's type.
|
||||
func ErrType(e error) ErrorType {
|
||||
var xe *xdsClientError
|
||||
if ok := errors.As(e, &xe); ok {
|
||||
return xe.t
|
||||
}
|
||||
return ErrorTypeUnknown
|
||||
}
|
||||
127
internal/xds/clients/xdsclient/internal/xdsresource/name.go
Normal file
127
internal/xds/clients/xdsclient/internal/xdsresource/name.go
Normal file
@@ -0,0 +1,127 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2021 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package xdsresource
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// FederationScheme is the scheme of a federation resource name.
|
||||
const FederationScheme = "xdstp"
|
||||
|
||||
// Name contains the parsed component of an xDS resource name.
|
||||
//
|
||||
// An xDS resource name is in the format of
|
||||
// xdstp://[{authority}]/{resource type}/{id/*}?{context parameters}{#processing directive,*}
|
||||
//
|
||||
// See
|
||||
// https://github.com/cncf/xds/blob/main/proposals/TP1-xds-transport-next.md#uri-based-xds-resource-names
|
||||
// for details, and examples.
|
||||
type Name struct {
|
||||
Scheme string
|
||||
Authority string
|
||||
Type string
|
||||
ID string
|
||||
|
||||
ContextParams map[string]string
|
||||
|
||||
processingDirective string
|
||||
}
|
||||
|
||||
// ParseName splits the name and returns a struct representation of the Name.
|
||||
//
|
||||
// If the name isn't a valid new-style xDS name, field ID is set to the input.
|
||||
// Note that this is not an error, because we still support the old-style
|
||||
// resource names (those not starting with "xdstp:").
|
||||
//
|
||||
// The caller can tell if the parsing is successful by checking the returned
|
||||
// Scheme.
|
||||
func ParseName(name string) *Name {
|
||||
if !strings.Contains(name, "://") {
|
||||
// Only the long form URL, with ://, is valid.
|
||||
return &Name{ID: name}
|
||||
}
|
||||
parsed, err := url.Parse(name)
|
||||
if err != nil {
|
||||
return &Name{ID: name}
|
||||
}
|
||||
|
||||
ret := &Name{
|
||||
Scheme: parsed.Scheme,
|
||||
Authority: parsed.Host,
|
||||
}
|
||||
split := strings.SplitN(parsed.Path, "/", 3)
|
||||
if len(split) < 3 {
|
||||
// Path is in the format of "/type/id". There must be at least 3
|
||||
// segments after splitting.
|
||||
return &Name{ID: name}
|
||||
}
|
||||
ret.Type = split[1]
|
||||
ret.ID = split[2]
|
||||
if len(parsed.Query()) != 0 {
|
||||
ret.ContextParams = make(map[string]string)
|
||||
for k, vs := range parsed.Query() {
|
||||
if len(vs) > 0 {
|
||||
// We only keep one value of each key. Behavior for multiple values
|
||||
// is undefined.
|
||||
ret.ContextParams[k] = vs[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
// TODO: processing directive (the part comes after "#" in the URL, stored
|
||||
// in parsed.RawFragment) is kept but not processed. Add support for that
|
||||
// when it's needed.
|
||||
ret.processingDirective = parsed.RawFragment
|
||||
return ret
|
||||
}
|
||||
|
||||
// String returns a canonicalized string of name. The context parameters are
|
||||
// sorted by the keys.
|
||||
func (n *Name) String() string {
|
||||
if n.Scheme == "" {
|
||||
return n.ID
|
||||
}
|
||||
|
||||
// Sort and build query.
|
||||
keys := make([]string, 0, len(n.ContextParams))
|
||||
for k := range n.ContextParams {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var pairs []string
|
||||
for _, k := range keys {
|
||||
pairs = append(pairs, strings.Join([]string{k, n.ContextParams[k]}, "="))
|
||||
}
|
||||
rawQuery := strings.Join(pairs, "&")
|
||||
|
||||
path := n.Type
|
||||
if n.ID != "" {
|
||||
path = "/" + path + "/" + n.ID
|
||||
}
|
||||
|
||||
tempURL := &url.URL{
|
||||
Scheme: n.Scheme,
|
||||
Host: n.Authority,
|
||||
Path: path,
|
||||
RawQuery: rawQuery,
|
||||
RawFragment: n.processingDirective,
|
||||
}
|
||||
return tempURL.String()
|
||||
}
|
||||
101
internal/xds/clients/xdsclient/internal/xdsresource/type.go
Normal file
101
internal/xds/clients/xdsclient/internal/xdsresource/type.go
Normal file
@@ -0,0 +1,101 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2021 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package xdsresource
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
|
||||
)
|
||||
|
||||
// UpdateMetadata contains the metadata for each update, including timestamp,
|
||||
// raw message, and so on.
|
||||
type UpdateMetadata struct {
|
||||
// Status is the status of this resource, e.g. ACKed, NACKed, or
|
||||
// Not_exist(removed).
|
||||
Status ServiceStatus
|
||||
// Version is the version of the xds response. Note that this is the version
|
||||
// of the resource in use (previous ACKed). If a response is NACKed, the
|
||||
// NACKed version is in ErrState.
|
||||
Version string
|
||||
// Timestamp is when the response is received.
|
||||
Timestamp time.Time
|
||||
// ErrState is set when the update is NACKed.
|
||||
ErrState *UpdateErrorMetadata
|
||||
}
|
||||
|
||||
// IsListenerResource returns true if the provider URL corresponds to an xDS
|
||||
// Listener resource.
|
||||
func IsListenerResource(url string) bool {
|
||||
return url == V3ListenerURL
|
||||
}
|
||||
|
||||
// IsHTTPConnManagerResource returns true if the provider URL corresponds to an xDS
|
||||
// HTTPConnManager resource.
|
||||
func IsHTTPConnManagerResource(url string) bool {
|
||||
return url == V3HTTPConnManagerURL
|
||||
}
|
||||
|
||||
// UnwrapResource unwraps and returns the inner resource if it's in a resource
|
||||
// wrapper. The original resource is returned if it's not wrapped.
|
||||
func UnwrapResource(r *anypb.Any) (*anypb.Any, error) {
|
||||
url := r.GetTypeUrl()
|
||||
if url != V3ResourceWrapperURL {
|
||||
// Not wrapped.
|
||||
return r, nil
|
||||
}
|
||||
inner := &v3discoverypb.Resource{}
|
||||
if err := proto.Unmarshal(r.GetValue(), inner); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return inner.Resource, nil
|
||||
}
|
||||
|
||||
// ServiceStatus is the status of the update.
|
||||
type ServiceStatus int
|
||||
|
||||
const (
|
||||
// ServiceStatusUnknown is the default state, before a watch is started for
|
||||
// the resource.
|
||||
ServiceStatusUnknown ServiceStatus = iota
|
||||
// ServiceStatusRequested is when the watch is started, but before and
|
||||
// response is received.
|
||||
ServiceStatusRequested
|
||||
// ServiceStatusNotExist is when the resource doesn't exist in
|
||||
// state-of-the-world responses (e.g. LDS and CDS), which means the resource
|
||||
// is removed by the management server.
|
||||
ServiceStatusNotExist // Resource is removed in the server, in LDS/CDS.
|
||||
// ServiceStatusACKed is when the resource is ACKed.
|
||||
ServiceStatusACKed
|
||||
// ServiceStatusNACKed is when the resource is NACKed.
|
||||
ServiceStatusNACKed
|
||||
)
|
||||
|
||||
// UpdateErrorMetadata is part of UpdateMetadata. It contains the error state
|
||||
// when a response is NACKed.
|
||||
type UpdateErrorMetadata struct {
|
||||
// Version is the version of the NACKed response.
|
||||
Version string
|
||||
// Err contains why the response was NACKed.
|
||||
Err error
|
||||
// Timestamp is when the NACKed response was received.
|
||||
Timestamp time.Time
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
// Package xdsresource defines constants to distinguish between supported xDS
|
||||
// API versions.
|
||||
package xdsresource
|
||||
|
||||
// Resource URLs. We need to be able to accept either version of the resource
|
||||
// regardless of the version of the transport protocol in use.
|
||||
const (
|
||||
googleapiPrefix = "type.googleapis.com/"
|
||||
|
||||
V3ListenerURL = googleapiPrefix + "envoy.config.listener.v3.Listener"
|
||||
V3HTTPConnManagerURL = googleapiPrefix + "envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager"
|
||||
V3ResourceWrapperURL = googleapiPrefix + "envoy.service.discovery.v3.Resource"
|
||||
)
|
||||
36
internal/xds/clients/xdsclient/logging.go
Normal file
36
internal/xds/clients/xdsclient/logging.go
Normal file
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"google.golang.org/grpc/grpclog"
|
||||
internalgrpclog "google.golang.org/grpc/internal/grpclog"
|
||||
)
|
||||
|
||||
var logger = grpclog.Component("xds")
|
||||
|
||||
func prefixLogger(p *XDSClient) *internalgrpclog.PrefixLogger {
|
||||
return internalgrpclog.NewPrefixLogger(logger, clientPrefix(p))
|
||||
}
|
||||
|
||||
func clientPrefix(p *XDSClient) string {
|
||||
return fmt.Sprintf("[xds-client %p] ", p)
|
||||
}
|
||||
64
internal/xds/clients/xdsclient/metrics/metrics.go
Normal file
64
internal/xds/clients/xdsclient/metrics/metrics.go
Normal file
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
// Package metrics defines all metrics that can be produced by an xDS client.
|
||||
// All calls to the MetricsRecorder by the xDS client will contain a struct
|
||||
// from this package passed by pointer.
|
||||
//
|
||||
// For definitions of these metrics and their labels, see gRFC A78:
|
||||
// https://github.com/grpc/proposal/blob/master/A78-grpc-metrics-wrr-xds.md
|
||||
package metrics
|
||||
|
||||
// ResourceUpdateValid is a metric to report a valid resource update from
|
||||
// the xDS management server for a given resource type.
|
||||
type ResourceUpdateValid struct {
|
||||
ServerURI string
|
||||
ResourceType string
|
||||
}
|
||||
|
||||
// ResourceUpdateInvalid is a metric to report an invalid resource update
|
||||
// from the xDS management server for a given resource type.
|
||||
type ResourceUpdateInvalid struct {
|
||||
ServerURI string
|
||||
ResourceType string
|
||||
}
|
||||
|
||||
// ServerFailure is a metric to report a server failure of the xDS
|
||||
// management server.
|
||||
type ServerFailure struct {
|
||||
ServerURI string
|
||||
}
|
||||
|
||||
// XDSClientConnected reports the connectivity state of the ADS stream.
|
||||
// Per gRFC A78, Value is 1 if connected, and 0 otherwise.
|
||||
// This metric provides the labels grpc.target and grpc.xds.server.
|
||||
type XDSClientConnected struct {
|
||||
ServerURI string
|
||||
Value int64
|
||||
}
|
||||
|
||||
// XDSClientResourceStats reports the current cache states of xDS resources.
|
||||
// For label definitions, see gRFC A78.
|
||||
// This metric provides the labels grpc.target, grpc.xds.authority,
|
||||
// grpc.xds.cache_state, and grpc.xds.resource_type.
|
||||
type XDSClientResourceStats struct {
|
||||
Authority string
|
||||
ResourceType string
|
||||
CacheState string
|
||||
Count int64
|
||||
}
|
||||
123
internal/xds/clients/xdsclient/resource_type.go
Normal file
123
internal/xds/clients/xdsclient/resource_type.go
Normal file
@@ -0,0 +1,123 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient
|
||||
|
||||
import "google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
// ResourceType wraps all resource-type specific functionality. Each supported
|
||||
// resource type needs to provide an implementation of the Decoder.
|
||||
type ResourceType struct {
|
||||
// TypeURL is the xDS type URL of this resource type for the v3 xDS
|
||||
// protocol. This URL is used as the key to look up the corresponding
|
||||
// ResourceType implementation in the ResourceTypes map provided in the
|
||||
// Config.
|
||||
TypeURL string
|
||||
|
||||
// TODO: Revisit if we need TypeURL to be part of the struct because it is
|
||||
// already a key in config's ResouceTypes map.
|
||||
|
||||
// TypeName is a shorter representation of the TypeURL to identify the
|
||||
// resource type. It is used for logging/debugging purposes.
|
||||
TypeName string
|
||||
|
||||
// AllResourcesRequiredInSotW indicates whether this resource type requires
|
||||
// that all resources be present in every SotW response from the server. If
|
||||
// true, a response that does not include a previously seen resource will
|
||||
// be interpreted as a deletion of that resource.
|
||||
AllResourcesRequiredInSotW bool
|
||||
|
||||
// Decoder is used to deserialize and validate an xDS resource received
|
||||
// from the xDS management server.
|
||||
Decoder Decoder
|
||||
}
|
||||
|
||||
// Decoder wraps the resource-type specific functionality for validation
|
||||
// and deserialization.
|
||||
type Decoder interface {
|
||||
// Decode deserializes and validates an xDS resource as received from the
|
||||
// xDS management server.
|
||||
//
|
||||
// The `resource` parameter may contain a value of the serialized wrapped
|
||||
// resource (i.e. with the type URL
|
||||
// `type.googleapis.com/envoy.service.discovery.v3.Resource`).
|
||||
// Implementations are responsible for unwrapping the underlying resource if
|
||||
// it is wrapped.
|
||||
//
|
||||
// If unmarshalling or validation fails, it returns a non-nil error.
|
||||
// Otherwise, returns a fully populated DecodeResult.
|
||||
Decode(resource *AnyProto, options DecodeOptions) (*DecodeResult, error)
|
||||
}
|
||||
|
||||
// AnyProto contains the type URL and serialized proto data of an xDS resource.
|
||||
type AnyProto struct {
|
||||
typeURL string
|
||||
value []byte
|
||||
}
|
||||
|
||||
// NewAnyProto creates an AnyProto from an anypb.Any. Must be called with a
|
||||
// non-nil argument.
|
||||
func NewAnyProto(a *anypb.Any) *AnyProto {
|
||||
return &AnyProto{
|
||||
typeURL: a.TypeUrl,
|
||||
value: a.Value,
|
||||
}
|
||||
}
|
||||
|
||||
// ToAny converts an AnyProto to an anypb.Any. Never returns nil.
|
||||
func (a *AnyProto) ToAny() *anypb.Any {
|
||||
return &anypb.Any{
|
||||
TypeUrl: a.typeURL,
|
||||
Value: a.value,
|
||||
}
|
||||
}
|
||||
|
||||
// DecodeOptions wraps the options required by ResourceType implementations for
|
||||
// decoding configuration received from the xDS management server.
|
||||
type DecodeOptions struct {
|
||||
// Config contains the complete configuration passed to the xDS client.
|
||||
// This contains useful data for resource validation.
|
||||
Config *Config
|
||||
|
||||
// ServerConfig contains the configuration of the xDS server that provided
|
||||
// the current resource being decoded.
|
||||
ServerConfig *ServerConfig
|
||||
}
|
||||
|
||||
// DecodeResult is the result of a decode operation.
|
||||
type DecodeResult struct {
|
||||
// Name is the name of the decoded resource.
|
||||
Name string
|
||||
|
||||
// Resource contains the configuration associated with the decoded
|
||||
// resource.
|
||||
Resource ResourceData
|
||||
}
|
||||
|
||||
// ResourceData contains the configuration data sent by the xDS management
|
||||
// server, associated with the resource being watched. Every resource type must
|
||||
// provide an implementation of this interface to represent the configuration
|
||||
// received from the xDS management server.
|
||||
type ResourceData interface {
|
||||
// Equal returns true if the passed in resource data is equal to that of
|
||||
// the receiver.
|
||||
Equal(other ResourceData) bool
|
||||
|
||||
// Bytes returns the underlying raw bytes of the resource proto.
|
||||
Bytes() []byte
|
||||
}
|
||||
48
internal/xds/clients/xdsclient/resource_watcher.go
Normal file
48
internal/xds/clients/xdsclient/resource_watcher.go
Normal file
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient
|
||||
|
||||
// ResourceWatcher is notified of the resource updates and errors that are
|
||||
// received by the xDS client from the management server.
|
||||
//
|
||||
// All methods on this interface are guaranteed to be called serially by the xDS
|
||||
// client.
|
||||
//
|
||||
// All methods contain a done parameter which should be called when processing
|
||||
// of the update has completed. For example, if processing a resource requires
|
||||
// watching new resources, those watches should be completed before done is
|
||||
// called, which can happen after the ResourceWatcher method has returned.
|
||||
// Failure to call done will prevent the xDS client from providing future
|
||||
// ResourceWatcher notifications.
|
||||
type ResourceWatcher interface {
|
||||
// ResourceChanged indicates a new version of the resource is available.
|
||||
ResourceChanged(resourceData ResourceData, done func())
|
||||
|
||||
// ResourceError indicates an error occurred while trying to fetch or
|
||||
// decode the associated resource. The previous version of the resource
|
||||
// should be considered invalid.
|
||||
ResourceError(err error, done func())
|
||||
|
||||
// AmbientError indicates an error occurred after a resource has been
|
||||
// received that should not modify the use of that resource but may provide
|
||||
// useful information about the state of the XDSClient for debugging
|
||||
// purposes. The previous version of the resource should still be
|
||||
// considered valid.
|
||||
AmbientError(err error, done func())
|
||||
}
|
||||
671
internal/xds/clients/xdsclient/test/ads_stream_ack_nack_test.go
Normal file
671
internal/xds/clients/xdsclient/test/ads_stream_ack_nack_test.go
Normal file
@@ -0,0 +1,671 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2024 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/uuid"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
"google.golang.org/grpc/internal/xds/clients/grpctransport"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/testing/protocmp"
|
||||
|
||||
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
|
||||
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
|
||||
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
|
||||
)
|
||||
|
||||
// Creates an xDS client with the given management server address, node ID
|
||||
// and transport builder.
|
||||
func createXDSClient(t *testing.T, mgmtServerAddress string, nodeID string, transportBuilder clients.TransportBuilder) *xdsclient.XDSClient {
|
||||
t.Helper()
|
||||
|
||||
resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
|
||||
si := clients.ServerIdentifier{
|
||||
ServerURI: mgmtServerAddress,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}},
|
||||
Node: clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
|
||||
TransportBuilder: transportBuilder,
|
||||
ResourceTypes: resourceTypes,
|
||||
// Xdstp resource names used in this test do not specify an
|
||||
// authority. These will end up looking up an entry with the
|
||||
// empty key in the authorities map. Having an entry with an
|
||||
// empty key and empty configuration, results in these
|
||||
// resources also using the top-level configuration.
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
"": {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
}
|
||||
|
||||
// Create an xDS client with the above config.
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { client.Close() })
|
||||
return client
|
||||
}
|
||||
|
||||
// Tests simple ACK and NACK scenarios on the ADS stream:
|
||||
// 1. When a good response is received, i.e. once that is expected to be ACKed,
|
||||
// the test verifies that an ACK is sent matching the version and nonce from
|
||||
// the response.
|
||||
// 2. When a subsequent bad response is received, i.e. once is expected to be
|
||||
// NACKed, the test verifies that a NACK is sent matching the previously
|
||||
// ACKed version and current nonce from the response.
|
||||
// 3. When a subsequent good response is received, the test verifies that an
|
||||
// ACK is sent matching the version and nonce from the current response.
|
||||
func (s) TestADS_ACK_NACK_Simple(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Create an xDS management server listening on a local port. Configure the
|
||||
// request and response handlers to push on channels that are inspected by
|
||||
// the test goroutine to verify ACK version and nonce.
|
||||
streamRequestCh := testutils.NewChannelWithSize(1)
|
||||
streamResponseCh := testutils.NewChannelWithSize(1)
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
|
||||
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
|
||||
streamRequestCh.SendContext(ctx, req)
|
||||
return nil
|
||||
},
|
||||
OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
|
||||
streamResponseCh.SendContext(ctx, resp)
|
||||
},
|
||||
})
|
||||
|
||||
// Create a listener resource on the management server.
|
||||
const listenerName = "listener"
|
||||
const routeConfigName = "route-config"
|
||||
nodeID := uuid.New().String()
|
||||
listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName)
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{listenerResource},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create an xDS client pointing to the above server.
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
|
||||
|
||||
// Register a watch for a listener resource.
|
||||
lw := newListenerWatcher()
|
||||
ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
|
||||
defer ldsCancel()
|
||||
|
||||
// Verify that the initial discovery request matches expectation.
|
||||
r, err := streamRequestCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Timeout when waiting for the initial discovery request")
|
||||
}
|
||||
gotReq := r.(*v3discoverypb.DiscoveryRequest)
|
||||
wantReq := &v3discoverypb.DiscoveryRequest{
|
||||
VersionInfo: "",
|
||||
Node: &v3corepb.Node{
|
||||
Id: nodeID,
|
||||
UserAgentName: "user-agent",
|
||||
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
|
||||
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
|
||||
},
|
||||
ResourceNames: []string{listenerName},
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
ResponseNonce: "",
|
||||
}
|
||||
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
|
||||
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
|
||||
}
|
||||
|
||||
// Capture the version and nonce from the response.
|
||||
r, err = streamResponseCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Timeout when waiting for a discovery response from the server")
|
||||
}
|
||||
gotResp := r.(*v3discoverypb.DiscoveryResponse)
|
||||
|
||||
// Verify that the ACK contains the appropriate version and nonce.
|
||||
r, err = streamRequestCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Timeout when waiting for ACK")
|
||||
}
|
||||
gotReq = r.(*v3discoverypb.DiscoveryRequest)
|
||||
wantReq.VersionInfo = gotResp.GetVersionInfo()
|
||||
wantReq.ResponseNonce = gotResp.GetNonce()
|
||||
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
|
||||
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
|
||||
}
|
||||
|
||||
// Verify the update received by the watcher.
|
||||
wantUpdate := listenerUpdateErrTuple{
|
||||
update: listenerUpdate{RouteConfigName: routeConfigName},
|
||||
}
|
||||
if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Update the management server with a listener resource that contains an
|
||||
// empty HTTP connection manager within the apiListener, which will cause
|
||||
// the resource to be NACKed.
|
||||
badListener := proto.Clone(listenerResource).(*v3listenerpb.Listener)
|
||||
badListener.ApiListener.ApiListener = nil
|
||||
mgmtServer.Update(ctx, e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{badListener},
|
||||
SkipValidation: true,
|
||||
})
|
||||
|
||||
r, err = streamResponseCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Timeout when waiting for a discovery response from the server")
|
||||
}
|
||||
gotResp = r.(*v3discoverypb.DiscoveryResponse)
|
||||
|
||||
wantNackErr := xdsresource.NewError(xdsresource.ErrorTypeNACKed, "unexpected http connection manager resource type")
|
||||
if err := verifyListenerUpdate(ctx, lw.ambientErrCh, listenerUpdateErrTuple{ambientErr: wantNackErr}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify that the NACK contains the appropriate version, nonce and error.
|
||||
// We expect the version to not change as this is a NACK.
|
||||
r, err = streamRequestCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Timeout when waiting for NACK")
|
||||
}
|
||||
gotReq = r.(*v3discoverypb.DiscoveryRequest)
|
||||
if gotNonce, wantNonce := gotReq.GetResponseNonce(), gotResp.GetNonce(); gotNonce != wantNonce {
|
||||
t.Errorf("Unexpected nonce in discovery request, got: %v, want: %v", gotNonce, wantNonce)
|
||||
}
|
||||
if gotErr := gotReq.GetErrorDetail(); gotErr == nil || !strings.Contains(gotErr.GetMessage(), wantNackErr.Error()) {
|
||||
t.Fatalf("Unexpected error in discovery request, got: %v, want: %v", gotErr.GetMessage(), wantNackErr)
|
||||
}
|
||||
|
||||
// Update the management server to send a good resource again.
|
||||
mgmtServer.Update(ctx, e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{listenerResource},
|
||||
SkipValidation: true,
|
||||
})
|
||||
|
||||
// The envoy-go-control-plane management server keeps resending the same
|
||||
// resource as long as we keep NACK'ing it. So, we will see the bad resource
|
||||
// sent to us a few times here, before receiving the good resource.
|
||||
var lastErr error
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
t.Fatalf("Timeout when waiting for an ACK from the xDS client. Last seen error: %v", lastErr)
|
||||
}
|
||||
|
||||
r, err = streamResponseCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Timeout when waiting for a discovery response from the server")
|
||||
}
|
||||
gotResp = r.(*v3discoverypb.DiscoveryResponse)
|
||||
|
||||
// Verify that the ACK contains the appropriate version and nonce.
|
||||
r, err = streamRequestCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Timeout when waiting for ACK")
|
||||
}
|
||||
gotReq = r.(*v3discoverypb.DiscoveryRequest)
|
||||
wantReq.VersionInfo = gotResp.GetVersionInfo()
|
||||
wantReq.ResponseNonce = gotResp.GetNonce()
|
||||
wantReq.ErrorDetail = nil
|
||||
diff := cmp.Diff(gotReq, wantReq, protocmp.Transform())
|
||||
if diff == "" {
|
||||
lastErr = nil
|
||||
break
|
||||
}
|
||||
lastErr = fmt.Errorf("unexpected diff in discovery request, diff (-got, +want):\n%s", diff)
|
||||
}
|
||||
|
||||
// Verify the update received by the watcher.
|
||||
for ; ctx.Err() == nil; <-time.After(100 * time.Millisecond) {
|
||||
if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
t.Fatalf("Timeout when waiting for listener update. Last seen error: %v", lastErr)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests the case where the first response is invalid. The test verifies that
|
||||
// the NACK contains an empty version string.
|
||||
func (s) TestADS_NACK_InvalidFirstResponse(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Create an xDS management server listening on a local port. Configure the
|
||||
// request and response handlers to push on channels that are inspected by
|
||||
// the test goroutine to verify ACK version and nonce.
|
||||
streamRequestCh := testutils.NewChannelWithSize(1)
|
||||
streamResponseCh := testutils.NewChannelWithSize(1)
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
|
||||
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
|
||||
streamRequestCh.SendContext(ctx, req)
|
||||
return nil
|
||||
},
|
||||
OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
|
||||
streamResponseCh.SendContext(ctx, resp)
|
||||
},
|
||||
})
|
||||
|
||||
// Create a listener resource on the management server that is expected to
|
||||
// be NACKed by the xDS client.
|
||||
const listenerName = "listener"
|
||||
const routeConfigName = "route-config"
|
||||
nodeID := uuid.New().String()
|
||||
listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName)
|
||||
listenerResource.ApiListener.ApiListener = nil
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{listenerResource},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create an xDS client pointing to the above server.
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
|
||||
|
||||
// Register a watch for a listener resource.
|
||||
lw := newListenerWatcher()
|
||||
ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
|
||||
defer ldsCancel()
|
||||
|
||||
// Verify that the initial discovery request matches expectation.
|
||||
r, err := streamRequestCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Timeout when waiting for the initial discovery request")
|
||||
}
|
||||
gotReq := r.(*v3discoverypb.DiscoveryRequest)
|
||||
wantReq := &v3discoverypb.DiscoveryRequest{
|
||||
VersionInfo: "",
|
||||
Node: &v3corepb.Node{
|
||||
Id: nodeID,
|
||||
UserAgentName: "user-agent",
|
||||
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
|
||||
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
|
||||
},
|
||||
ResourceNames: []string{listenerName},
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
ResponseNonce: "",
|
||||
}
|
||||
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
|
||||
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
|
||||
}
|
||||
|
||||
// Capture the version and nonce from the response.
|
||||
r, err = streamResponseCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Timeout when waiting for the discovery response from client")
|
||||
}
|
||||
gotResp := r.(*v3discoverypb.DiscoveryResponse)
|
||||
|
||||
// Verify that the error is propagated to the watcher.
|
||||
var wantNackErr = xdsresource.NewError(xdsresource.ErrorTypeNACKed, "unexpected http connection manager resource type")
|
||||
if err := verifyListenerUpdate(ctx, lw.resourceErrCh, listenerUpdateErrTuple{resourceErr: wantNackErr}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// NACK should contain the appropriate error, nonce, but empty version.
|
||||
r, err = streamRequestCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Timeout when waiting for ACK")
|
||||
}
|
||||
gotReq = r.(*v3discoverypb.DiscoveryRequest)
|
||||
if gotVersion, wantVersion := gotReq.GetVersionInfo(), ""; gotVersion != wantVersion {
|
||||
t.Errorf("Unexpected version in discovery request, got: %v, want: %v", gotVersion, wantVersion)
|
||||
}
|
||||
if gotNonce, wantNonce := gotReq.GetResponseNonce(), gotResp.GetNonce(); gotNonce != wantNonce {
|
||||
t.Errorf("Unexpected nonce in discovery request, got: %v, want: %v", gotNonce, wantNonce)
|
||||
}
|
||||
if gotErr := gotReq.GetErrorDetail(); gotErr == nil || !strings.Contains(gotErr.GetMessage(), wantNackErr.Error()) {
|
||||
t.Fatalf("Unexpected error in discovery request, got: %v, want: %v", gotErr.GetMessage(), wantNackErr)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests the scenario where the xDS client is no longer interested in a
|
||||
// resource. The following sequence of events are tested:
|
||||
// 1. A resource is requested and a good response is received. The test verifies
|
||||
// that an ACK is sent for this resource.
|
||||
// 2. The previously requested resource is no longer requested. The test
|
||||
// verifies that the connection to the management server is closed.
|
||||
// 3. The same resource is requested again. The test verifies that a new
|
||||
// request is sent with an empty version string, which corresponds to the
|
||||
// first request on a new connection.
|
||||
func (s) TestADS_ACK_NACK_ResourceIsNotRequestedAnymore(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Create an xDS management server listening on a local port. Configure the
|
||||
// request and response handlers to push on channels that are inspected by
|
||||
// the test goroutine to verify ACK version and nonce.
|
||||
streamRequestCh := testutils.NewChannelWithSize(1)
|
||||
streamResponseCh := testutils.NewChannelWithSize(1)
|
||||
streamCloseCh := testutils.NewChannelWithSize(1)
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
|
||||
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
|
||||
streamRequestCh.SendContext(ctx, req)
|
||||
return nil
|
||||
},
|
||||
OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
|
||||
streamResponseCh.SendContext(ctx, resp)
|
||||
},
|
||||
OnStreamClosed: func(int64, *v3corepb.Node) {
|
||||
streamCloseCh.SendContext(ctx, struct{}{})
|
||||
},
|
||||
})
|
||||
|
||||
// Create a listener resource on the management server.
|
||||
const listenerName = "listener"
|
||||
const routeConfigName = "route-config"
|
||||
nodeID := uuid.New().String()
|
||||
listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName)
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{listenerResource},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create an xDS client pointing to the above server.
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
|
||||
|
||||
// Register a watch for a listener resource.
|
||||
lw := newListenerWatcher()
|
||||
ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
|
||||
defer ldsCancel()
|
||||
|
||||
// Verify that the initial discovery request matches expectation.
|
||||
r, err := streamRequestCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Timeout when waiting for the initial discovery request")
|
||||
}
|
||||
gotReq := r.(*v3discoverypb.DiscoveryRequest)
|
||||
wantReq := &v3discoverypb.DiscoveryRequest{
|
||||
VersionInfo: "",
|
||||
Node: &v3corepb.Node{
|
||||
Id: nodeID,
|
||||
UserAgentName: "user-agent",
|
||||
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
|
||||
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
|
||||
},
|
||||
ResourceNames: []string{listenerName},
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
ResponseNonce: "",
|
||||
}
|
||||
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
|
||||
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
|
||||
}
|
||||
|
||||
// Capture the version and nonce from the response.
|
||||
r, err = streamResponseCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Timeout when waiting for the discovery response from client")
|
||||
}
|
||||
gotResp := r.(*v3discoverypb.DiscoveryResponse)
|
||||
|
||||
// Verify that the ACK contains the appropriate version and nonce.
|
||||
r, err = streamRequestCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Timeout when waiting for ACK")
|
||||
}
|
||||
gotReq = r.(*v3discoverypb.DiscoveryRequest)
|
||||
wantACKReq := proto.Clone(wantReq).(*v3discoverypb.DiscoveryRequest)
|
||||
wantACKReq.VersionInfo = gotResp.GetVersionInfo()
|
||||
wantACKReq.ResponseNonce = gotResp.GetNonce()
|
||||
if diff := cmp.Diff(gotReq, wantACKReq, protocmp.Transform()); diff != "" {
|
||||
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
|
||||
}
|
||||
|
||||
// Verify the update received by the watcher.
|
||||
wantUpdate := listenerUpdateErrTuple{
|
||||
update: listenerUpdate{RouteConfigName: routeConfigName},
|
||||
}
|
||||
if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Cancel the watch on the listener resource. This should result in the
|
||||
// existing connection to be management server getting closed.
|
||||
ldsCancel()
|
||||
if _, err := streamCloseCh.Receive(ctx); err != nil {
|
||||
t.Fatalf("Timeout when expecting existing connection to be closed: %v", err)
|
||||
}
|
||||
|
||||
// There is a race between two events when the last watch on an xdsChannel
|
||||
// is canceled:
|
||||
// - an empty discovery request being sent out
|
||||
// - the ADS stream being closed
|
||||
// To handle this race, we drain the request channel here so that if an
|
||||
// empty discovery request was received, it is pulled out of the request
|
||||
// channel and thereby guaranteeing a clean slate for the next watch
|
||||
// registered below.
|
||||
streamRequestCh.Drain()
|
||||
|
||||
// Register a watch for the same listener resource.
|
||||
lw = newListenerWatcher()
|
||||
ldsCancel = client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
|
||||
defer ldsCancel()
|
||||
|
||||
// Verify that the discovery request is identical to the first one sent out
|
||||
// to the management server.
|
||||
r, err = streamRequestCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatal("Timeout when waiting for discovery request")
|
||||
}
|
||||
gotReq = r.(*v3discoverypb.DiscoveryRequest)
|
||||
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
|
||||
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
|
||||
}
|
||||
|
||||
// Verify the update received by the watcher.
|
||||
if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestADS_NACKError_DuplicateSuppression verifies that when a duplicate invalid
|
||||
// LDS resource is sent by the server, the client suppresses the duplicate error
|
||||
// notification to the watcher.
|
||||
func (s) TestADS_NACKError_DuplicateSuppression(t *testing.T) {
|
||||
// The first NACK is expected upon receiving the invalid resource. The
|
||||
// second NACK is a duplicate, sent because the management server resends
|
||||
// the invalid resource in a loop as long as the resource update has not
|
||||
// changed to a valid one. The third NACK is waited for to verify that the
|
||||
// duplicated error is still suppressed as expected.
|
||||
const wantNACKCount = 3
|
||||
|
||||
nacksReceivedCh := make(chan struct{})
|
||||
nackCount := 0
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
|
||||
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
|
||||
if req.GetTypeUrl() != xdsresource.V3ListenerURL {
|
||||
return nil
|
||||
}
|
||||
if req.GetErrorDetail() != nil {
|
||||
nackCount++
|
||||
if nackCount == wantNACKCount {
|
||||
close(nacksReceivedCh)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
const listenerName = "listener"
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
// Create an xDS client pointing to the above server.
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
|
||||
|
||||
watcher := newListenerWatcher()
|
||||
ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher)
|
||||
defer ldsCancel()
|
||||
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{badListenerResource(t, listenerName)},
|
||||
SkipValidation: true,
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
|
||||
// Verify that the watcher receives the initial error.
|
||||
v, err := watcher.resourceErrCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Timeout when waiting for an endpoints resource error from the management server: %v", err)
|
||||
}
|
||||
const wantNackErr = "no RouteSpecifier"
|
||||
if gotErr := v.(listenerUpdateErrTuple).resourceErr; !strings.Contains(gotErr.Error(), wantNackErr) {
|
||||
t.Fatalf("Update received with error: %v, want %q", gotErr, wantNackErr)
|
||||
}
|
||||
|
||||
// Wait for the management server to receive at least wantNACKCount NACKs,
|
||||
// indicating that the NACK resend loop has run multiple times.
|
||||
select {
|
||||
case <-nacksReceivedCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timeout waiting for %d NACKs to be received by the server: %v", wantNACKCount, ctx.Err())
|
||||
}
|
||||
|
||||
// Verify that no duplicate error update was written to the error channel,
|
||||
// proving that subsequent duplicate error updates were suppressed.
|
||||
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
||||
defer sCancel()
|
||||
if v, err := watcher.resourceErrCh.Receive(sCtx); err == nil {
|
||||
t.Fatalf("Unexpected duplicate error received by watcher: %v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// TestADS_NACKError_DuplicateSuppression_MultipleResources verifies that
|
||||
// when multiple invalid resources cause errors, and a subsequent update carries
|
||||
// a duplicate error for only a subset of resources, the client still suppresses
|
||||
// the duplicate error notification to the watcher of the affected resource(s).
|
||||
func (s) TestADS_NACKError_DuplicateSuppression_MultipleResources(t *testing.T) {
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
|
||||
|
||||
const listenerName1 = "listener-1"
|
||||
const listenerName2 = "listener-2"
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
|
||||
|
||||
watcher1 := newListenerWatcher()
|
||||
ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, listenerName1, watcher1)
|
||||
defer ldsCancel1()
|
||||
|
||||
watcher2 := newListenerWatcher()
|
||||
ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, listenerName2, watcher2)
|
||||
defer ldsCancel2()
|
||||
|
||||
// Update 1: Management server sends two invalid listener resources.
|
||||
resources1 := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{
|
||||
badListenerResource(t, listenerName1),
|
||||
badListenerResource(t, listenerName2),
|
||||
},
|
||||
SkipValidation: true,
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := mgmtServer.Update(ctx, resources1); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources1, err)
|
||||
}
|
||||
|
||||
// Verify that watcher1 receives the initial error for listener 1.
|
||||
v1, err := watcher1.resourceErrCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Timeout waiting for listener 1 error from management server: %v", err)
|
||||
}
|
||||
const wantNackErr = "no RouteSpecifier"
|
||||
if gotErr := v1.(listenerUpdateErrTuple).resourceErr; !strings.Contains(gotErr.Error(), wantNackErr) {
|
||||
t.Fatalf("Update 1 listener 1 received with error: %v, want %q", gotErr, wantNackErr)
|
||||
}
|
||||
|
||||
// Verify that watcher2 receives the initial error for listener 2.
|
||||
v2, err := watcher2.resourceErrCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Timeout waiting for listener 2 error from management server: %v", err)
|
||||
}
|
||||
|
||||
if gotErr := v2.(listenerUpdateErrTuple).resourceErr; !strings.Contains(gotErr.Error(), wantNackErr) {
|
||||
t.Fatalf("Update 1 listener 2 received with error: %v, want %q", gotErr, wantNackErr)
|
||||
}
|
||||
|
||||
// Update 2: Management server sends listener 1 still invalid, but listener
|
||||
// 2 now valid.
|
||||
resources2 := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{
|
||||
badListenerResource(t, listenerName1),
|
||||
e2e.DefaultClientListener(listenerName2, "route-config-2"),
|
||||
},
|
||||
SkipValidation: true,
|
||||
}
|
||||
|
||||
if err := mgmtServer.Update(ctx, resources2); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources2, err)
|
||||
}
|
||||
|
||||
// Wait for watcher2 to receive the valid resource update for listener 2.
|
||||
if _, err := watcher2.updateCh.Receive(ctx); err != nil {
|
||||
t.Fatalf("Timeout waiting for listener 2 resource update: %v", err)
|
||||
}
|
||||
|
||||
// Verify that no duplicate error update was written to watcher 1.
|
||||
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
||||
defer sCancel()
|
||||
if v, err := watcher1.resourceErrCh.Receive(sCtx); err == nil {
|
||||
t.Fatalf("Unexpected duplicate error received by watcher 1: %v", v)
|
||||
}
|
||||
}
|
||||
444
internal/xds/clients/xdsclient/test/ads_stream_backoff_test.go
Normal file
444
internal/xds/clients/xdsclient/test/ads_stream_backoff_test.go
Normal file
@@ -0,0 +1,444 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2024 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/internal/xds/clients/grpctransport"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient"
|
||||
xdsclientinternal "google.golang.org/grpc/internal/xds/clients/xdsclient/internal"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
|
||||
"google.golang.org/protobuf/testing/protocmp"
|
||||
|
||||
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
|
||||
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
|
||||
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func overrideStreamBackOff(t *testing.T, streamBackOff func(int) time.Duration) {
|
||||
originalStreamBackoff := xdsclientinternal.StreamBackoff
|
||||
xdsclientinternal.StreamBackoff = streamBackOff
|
||||
t.Cleanup(func() { xdsclientinternal.StreamBackoff = originalStreamBackoff })
|
||||
}
|
||||
|
||||
// Creates an xDS client with the given management server address, nodeID and backoff function.
|
||||
func createXDSClientWithBackoff(t *testing.T, mgmtServerAddress string, nodeID string, streamBackoff func(int) time.Duration) *xdsclient.XDSClient {
|
||||
t.Helper()
|
||||
overrideStreamBackOff(t, streamBackoff)
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
return createXDSClient(t, mgmtServerAddress, nodeID, grpctransport.NewBuilder(configs))
|
||||
}
|
||||
|
||||
// Tests the case where the management server returns an error in the ADS
|
||||
// streaming RPC. Verifies that the ADS stream is restarted after a backoff
|
||||
// period, and that the previously requested resources are re-requested on the
|
||||
// new stream.
|
||||
func (s) TestADS_BackoffAfterStreamFailure(t *testing.T) {
|
||||
// Channels used for verifying different events in the test.
|
||||
streamCloseCh := make(chan struct{}, 1) // ADS stream is closed.
|
||||
ldsResourcesCh := make(chan []string, 1) // Listener resource names in the discovery request.
|
||||
backoffCh := make(chan struct{}, 1) // Backoff after stream failure.
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Create an xDS management server that returns RPC errors.
|
||||
streamErr := errors.New("ADS stream error")
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
|
||||
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
|
||||
// Push the requested resource names on to a channel.
|
||||
if req.GetTypeUrl() == version.V3ListenerURL {
|
||||
t.Logf("Received LDS request for resources: %v", req.GetResourceNames())
|
||||
select {
|
||||
case ldsResourcesCh <- req.GetResourceNames():
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}
|
||||
// Return an error everytime a request is sent on the stream. This
|
||||
// should cause the transport to backoff before attempting to
|
||||
// recreate the stream.
|
||||
return streamErr
|
||||
},
|
||||
// Push on a channel whenever the stream is closed.
|
||||
OnStreamClosed: func(int64, *v3corepb.Node) {
|
||||
select {
|
||||
case streamCloseCh <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
// Override the backoff implementation to push on a channel that is read by
|
||||
// the test goroutine.
|
||||
backoffCtx, backoffCancel := context.WithCancel(ctx)
|
||||
streamBackoff := func(int) time.Duration {
|
||||
select {
|
||||
case backoffCh <- struct{}{}:
|
||||
case <-backoffCtx.Done():
|
||||
}
|
||||
return 0
|
||||
}
|
||||
defer backoffCancel()
|
||||
|
||||
// Create an xDS client with bootstrap pointing to the above server.
|
||||
nodeID := uuid.New().String()
|
||||
client := createXDSClientWithBackoff(t, mgmtServer.Address, nodeID, streamBackoff)
|
||||
|
||||
// Register a watch for a listener resource.
|
||||
const listenerName = "listener"
|
||||
lw := newListenerWatcher()
|
||||
ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
|
||||
defer ldsCancel()
|
||||
|
||||
// Verify that an ADS stream is created and an LDS request with the above
|
||||
// resource name is sent.
|
||||
if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify that the received stream error is reported to the watcher.
|
||||
if err := verifyListenerResourceError(ctx, lw.resourceErrCh, streamErr.Error(), nodeID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify that the stream is closed.
|
||||
select {
|
||||
case <-streamCloseCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timeout waiting for stream to be closed after an error")
|
||||
}
|
||||
|
||||
// Verify that the ADS stream backs off before recreating the stream.
|
||||
select {
|
||||
case <-backoffCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timeout waiting for ADS stream to backoff after stream failure")
|
||||
}
|
||||
|
||||
// Verify that the same resource name is re-requested on the new stream.
|
||||
if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// To prevent indefinite blocking during xDS client close, which is caused
|
||||
// by a blocking backoff channel write, cancel the backoff context early
|
||||
// given that the test is complete.
|
||||
backoffCancel()
|
||||
|
||||
}
|
||||
|
||||
// Tests the case where a stream breaks because the server goes down. Verifies
|
||||
// that when the server comes back up, the same resources are re-requested, this
|
||||
// time with the previously acked version and an empty nonce.
|
||||
func (s) TestADS_RetriesAfterBrokenStream(t *testing.T) {
|
||||
// Channels used for verifying different events in the test.
|
||||
streamRequestCh := make(chan *v3discoverypb.DiscoveryRequest, 1) // Discovery request is received.
|
||||
streamResponseCh := make(chan *v3discoverypb.DiscoveryResponse, 1) // Discovery response is received.
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Create an xDS management server listening on a local port.
|
||||
l, err := net.Listen("tcp", "localhost:0")
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen() failed: %v", err)
|
||||
}
|
||||
lis := testutils.NewRestartableListener(l)
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
|
||||
Listener: lis,
|
||||
// Push the received request on to a channel for the test goroutine to
|
||||
// verify that it matches expectations.
|
||||
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
|
||||
select {
|
||||
case streamRequestCh <- req:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
return nil
|
||||
},
|
||||
// Push the response that the management server is about to send on to a
|
||||
// channel. The test goroutine to uses this to extract the version and
|
||||
// nonce, expected on subsequent requests.
|
||||
OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
|
||||
select {
|
||||
case streamResponseCh <- resp:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
// Create a listener resource on the management server.
|
||||
const listenerName = "listener"
|
||||
const routeConfigName = "route-config"
|
||||
nodeID := uuid.New().String()
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeConfigName)},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Override the backoff implementation to always return 0, to reduce test
|
||||
// run time. Instead control when the backoff returns by blocking on a
|
||||
// channel, that the test closes.
|
||||
backoffCh := make(chan struct{})
|
||||
streamBackoff := func(int) time.Duration {
|
||||
select {
|
||||
case backoffCh <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Create an xDS client pointing to the above server.
|
||||
client := createXDSClientWithBackoff(t, mgmtServer.Address, nodeID, streamBackoff)
|
||||
|
||||
// Register a watch for a listener resource.
|
||||
lw := newListenerWatcher()
|
||||
ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
|
||||
defer ldsCancel()
|
||||
|
||||
// Verify that the initial discovery request matches expectation.
|
||||
var gotReq *v3discoverypb.DiscoveryRequest
|
||||
select {
|
||||
case gotReq = <-streamRequestCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timeout waiting for discovery request on the stream")
|
||||
}
|
||||
wantReq := &v3discoverypb.DiscoveryRequest{
|
||||
VersionInfo: "",
|
||||
Node: &v3corepb.Node{
|
||||
Id: nodeID,
|
||||
UserAgentName: "user-agent",
|
||||
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
|
||||
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
|
||||
},
|
||||
ResourceNames: []string{listenerName},
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
ResponseNonce: "",
|
||||
}
|
||||
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
|
||||
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
|
||||
}
|
||||
|
||||
// Capture the version and nonce from the response.
|
||||
var gotResp *v3discoverypb.DiscoveryResponse
|
||||
select {
|
||||
case gotResp = <-streamResponseCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timeout waiting for discovery response on the stream")
|
||||
}
|
||||
version := gotResp.GetVersionInfo()
|
||||
nonce := gotResp.GetNonce()
|
||||
|
||||
// Verify that the ACK contains the appropriate version and nonce.
|
||||
wantReq.VersionInfo = version
|
||||
wantReq.ResponseNonce = nonce
|
||||
select {
|
||||
case gotReq = <-streamRequestCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timeout waiting for the discovery request ACK on the stream")
|
||||
}
|
||||
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
|
||||
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
|
||||
}
|
||||
|
||||
// Verify the update received by the watcher.
|
||||
wantUpdate := listenerUpdateErrTuple{
|
||||
update: listenerUpdate{
|
||||
RouteConfigName: routeConfigName},
|
||||
}
|
||||
if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Bring down the management server to simulate a broken stream.
|
||||
lis.Stop()
|
||||
|
||||
// Verify that the error callback on the watcher is not invoked.
|
||||
verifyNoListenerUpdate(ctx, lw.updateCh)
|
||||
|
||||
// Wait for backoff to kick in, and unblock the first backoff attempt.
|
||||
select {
|
||||
case <-backoffCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Timeout waiting for stream backoff")
|
||||
}
|
||||
|
||||
// Bring up the management server. The test does not have prcecise control
|
||||
// over when new streams to the management server will start succeeding. The
|
||||
// ADS stream implementation will backoff as many times as required before
|
||||
// it can successfully create a new stream. Therefore, we need to receive on
|
||||
// the backoffCh as many times as required, and unblock the backoff
|
||||
// implementation.
|
||||
lis.Restart()
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-backoffCh:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Verify that the transport creates a new stream and sends out a new
|
||||
// request which contains the previously acked version, but an empty nonce.
|
||||
wantReq.ResponseNonce = ""
|
||||
select {
|
||||
case gotReq = <-streamRequestCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timeout waiting for the discovery request ACK on the stream")
|
||||
}
|
||||
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
|
||||
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests the case where a resource is requested before the a valid ADS stream
|
||||
// exists. Verifies that the a discovery request is sent out for the previously
|
||||
// requested resource once a valid stream is created.
|
||||
func (s) TestADS_ResourceRequestedBeforeStreamCreation(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Channels used for verifying different events in the test.
|
||||
streamRequestCh := make(chan *v3discoverypb.DiscoveryRequest, 1) // Discovery request is received.
|
||||
|
||||
// Create an xDS management server listening on a local port.
|
||||
l, err := net.Listen("tcp", "localhost:0")
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen() failed: %v", err)
|
||||
}
|
||||
lis := testutils.NewRestartableListener(l)
|
||||
streamErr := errors.New("ADS stream error")
|
||||
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
|
||||
Listener: lis,
|
||||
|
||||
// Return an error everytime a request is sent on the stream. This
|
||||
// should cause the transport to backoff before attempting to recreate
|
||||
// the stream.
|
||||
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
|
||||
select {
|
||||
case streamRequestCh <- req:
|
||||
default:
|
||||
}
|
||||
return streamErr
|
||||
},
|
||||
})
|
||||
|
||||
// Bring down the management server before creating the transport. This
|
||||
// allows us to test the case where SendRequest() is called when there is no
|
||||
// stream to the management server.
|
||||
lis.Stop()
|
||||
|
||||
// Override the backoff implementation to always return 0, to reduce test
|
||||
// run time. Instead control when the backoff returns by blocking on a
|
||||
// channel, that the test closes.
|
||||
backoffCh := make(chan struct{}, 1)
|
||||
unblockBackoffCh := make(chan struct{})
|
||||
streamBackoff := func(int) time.Duration {
|
||||
select {
|
||||
case backoffCh <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
<-unblockBackoffCh
|
||||
return 0
|
||||
}
|
||||
|
||||
// Create an xDS client with bootstrap pointing to the above server.
|
||||
nodeID := uuid.New().String()
|
||||
client := createXDSClientWithBackoff(t, mgmtServer.Address, nodeID, streamBackoff)
|
||||
|
||||
// Register a watch for a listener resource.
|
||||
const listenerName = "listener"
|
||||
lw := newListenerWatcher()
|
||||
ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
|
||||
defer ldsCancel()
|
||||
|
||||
// The above watch results in an attempt to create a new stream, which will
|
||||
// fail, and will result in backoff. Wait for backoff to kick in.
|
||||
select {
|
||||
case <-backoffCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Timeout waiting for stream backoff")
|
||||
}
|
||||
|
||||
// Bring up the connection to the management server, and unblock the backoff
|
||||
// implementation.
|
||||
lis.Restart()
|
||||
close(unblockBackoffCh)
|
||||
|
||||
// Verify that the initial discovery request matches expectation.
|
||||
var gotReq *v3discoverypb.DiscoveryRequest
|
||||
select {
|
||||
case gotReq = <-streamRequestCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timeout waiting for discovery request on the stream")
|
||||
}
|
||||
wantReq := &v3discoverypb.DiscoveryRequest{
|
||||
VersionInfo: "",
|
||||
Node: &v3corepb.Node{
|
||||
Id: nodeID,
|
||||
UserAgentName: "user-agent",
|
||||
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
|
||||
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
|
||||
},
|
||||
ResourceNames: []string{listenerName},
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
ResponseNonce: "",
|
||||
}
|
||||
if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
|
||||
t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
// waitForResourceNames waits for the wantNames to be received on namesCh.
|
||||
// Returns a non-nil error if the context expires before that.
|
||||
func waitForResourceNames(ctx context.Context, t *testing.T, namesCh chan []string, wantNames []string) error {
|
||||
t.Helper()
|
||||
|
||||
var lastRequestedNames []string
|
||||
for ; ; <-time.After(defaultTestShortTimeout) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("timeout waiting for resources %v to be requested from the management server. Last requested resources: %v", wantNames, lastRequestedNames)
|
||||
case gotNames := <-namesCh:
|
||||
if cmp.Equal(gotNames, wantNames, cmpopts.EquateEmpty(), cmpopts.SortSlices(func(s1, s2 string) bool { return s1 < s2 })) {
|
||||
return nil
|
||||
}
|
||||
lastRequestedNames = gotNames
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,625 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2024 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
|
||||
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
|
||||
"github.com/google/uuid"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/internal/testutils/xds/e2e"
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
|
||||
)
|
||||
|
||||
// blockingListenerWatcher implements xdsresource.ListenerWatcher. It writes to
|
||||
// a channel when it receives a callback from the watch. It also makes the
|
||||
// DoneNotifier passed to the callback available to the test, thereby enabling
|
||||
// the test to block this watcher for as long as required.
|
||||
type blockingListenerWatcher struct {
|
||||
doneNotifierCh chan func() // DoneNotifier passed to the callback.
|
||||
updateCh chan struct{} // Written to when an update is received.
|
||||
ambientErrCh chan struct{} // Written to when an ambient error is received.
|
||||
resourceErrCh chan struct{} // Written to when a resource error is received.
|
||||
}
|
||||
|
||||
func newBLockingListenerWatcher() *blockingListenerWatcher {
|
||||
return &blockingListenerWatcher{
|
||||
doneNotifierCh: make(chan func(), 1),
|
||||
updateCh: make(chan struct{}, 1),
|
||||
ambientErrCh: make(chan struct{}, 1),
|
||||
resourceErrCh: make(chan struct{}, 1),
|
||||
}
|
||||
}
|
||||
|
||||
func (lw *blockingListenerWatcher) ResourceChanged(_ xdsclient.ResourceData, done func()) {
|
||||
// Notify receipt of the update.
|
||||
select {
|
||||
case lw.updateCh <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
|
||||
select {
|
||||
case lw.doneNotifierCh <- done:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (lw *blockingListenerWatcher) ResourceError(_ error, done func()) {
|
||||
// Notify receipt of an error.
|
||||
select {
|
||||
case lw.resourceErrCh <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
|
||||
select {
|
||||
case lw.doneNotifierCh <- done:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (lw *blockingListenerWatcher) AmbientError(_ error, done func()) {
|
||||
// Notify receipt of an error.
|
||||
select {
|
||||
case lw.ambientErrCh <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
|
||||
select {
|
||||
case lw.doneNotifierCh <- done:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
type transportBuilder struct {
|
||||
adsStreamCh chan *stream
|
||||
}
|
||||
|
||||
func (b *transportBuilder) Build(si clients.ServerIdentifier) (clients.Transport, error) {
|
||||
cc, err := grpc.NewClient(si.ServerURI, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultCallOptions(grpc.ForceCodec(&byteCodec{})))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &transport{cc: cc, adsStreamCh: b.adsStreamCh}, nil
|
||||
}
|
||||
|
||||
type transport struct {
|
||||
cc *grpc.ClientConn
|
||||
adsStreamCh chan *stream
|
||||
}
|
||||
|
||||
func (t *transport) NewStream(ctx context.Context, method string) (clients.Stream, error) {
|
||||
s, err := t.cc.NewStream(ctx, &grpc.StreamDesc{ClientStreams: true, ServerStreams: true}, method)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
stream := &stream{
|
||||
stream: s,
|
||||
recvCh: make(chan struct{}, 1),
|
||||
}
|
||||
t.adsStreamCh <- stream
|
||||
|
||||
return stream, nil
|
||||
}
|
||||
|
||||
func (t *transport) Close() {
|
||||
t.cc.Close()
|
||||
}
|
||||
|
||||
type stream struct {
|
||||
stream grpc.ClientStream
|
||||
recvCh chan struct{}
|
||||
}
|
||||
|
||||
func (s *stream) Send(msg []byte) error {
|
||||
return s.stream.SendMsg(msg)
|
||||
}
|
||||
|
||||
func (s *stream) Recv() ([]byte, error) {
|
||||
select {
|
||||
case s.recvCh <- struct{}{}:
|
||||
case <-s.stream.Context().Done():
|
||||
// Unblock the recv() once the stream is done.
|
||||
}
|
||||
|
||||
var typedRes []byte
|
||||
if err := s.stream.RecvMsg(&typedRes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return typedRes, nil
|
||||
}
|
||||
|
||||
type byteCodec struct{}
|
||||
|
||||
func (c *byteCodec) Marshal(v any) ([]byte, error) {
|
||||
if b, ok := v.([]byte); ok {
|
||||
return b, nil
|
||||
}
|
||||
return nil, fmt.Errorf("transport: message is %T, but must be a []byte", v)
|
||||
}
|
||||
|
||||
func (c *byteCodec) Unmarshal(data []byte, v any) error {
|
||||
if b, ok := v.(*[]byte); ok {
|
||||
*b = data
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("transport: target is %T, but must be *[]byte", v)
|
||||
}
|
||||
|
||||
func (c *byteCodec) Name() string {
|
||||
return "transport.byteCodec"
|
||||
}
|
||||
|
||||
// Tests ADS stream level flow control with a single resource. The test does the
|
||||
// following:
|
||||
// - Starts a management server and configures a listener resource on it.
|
||||
// - Creates an xDS client to the above management server, starts a couple of
|
||||
// listener watchers for the above resource, and verifies that the update
|
||||
// reaches these watchers.
|
||||
// - These watchers don't invoke the onDone callback until explicitly
|
||||
// triggered by the test. This allows the test to verify that the next
|
||||
// Recv() call on the ADS stream does not happen until both watchers have
|
||||
// completely processed the update, i.e invoke the onDone callback.
|
||||
// - Resource is updated on the management server, and the test verifies that
|
||||
// the update reaches the watchers.
|
||||
func (s) TestADSFlowControl_ResourceUpdates_SingleResource(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Start an xDS management server.
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
|
||||
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
// Create an xDS client pointing to the above server with a test transport
|
||||
// that allow monitoring the underlying stream through adsStreamCh.
|
||||
adsStreamCh := make(chan *stream, 1)
|
||||
client := createXDSClient(t, mgmtServer.Address, nodeID, &transportBuilder{adsStreamCh: adsStreamCh})
|
||||
|
||||
// Configure two watchers for the same listener resource.
|
||||
const listenerResourceName = "test-listener-resource"
|
||||
const routeConfigurationName = "test-route-configuration-resource"
|
||||
watcher1 := newBLockingListenerWatcher()
|
||||
cancel1 := client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName, watcher1)
|
||||
defer cancel1()
|
||||
watcher2 := newBLockingListenerWatcher()
|
||||
cancel2 := client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName, watcher2)
|
||||
defer cancel2()
|
||||
|
||||
// Wait for the ADS stream to be created.
|
||||
var adsStream *stream
|
||||
select {
|
||||
case adsStream = <-adsStreamCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for ADS stream to be created")
|
||||
}
|
||||
|
||||
// Configure the listener resource on the management server.
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
|
||||
// Ensure that there is a read on the stream.
|
||||
select {
|
||||
case <-adsStream.recvCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for ADS stream to be read from")
|
||||
}
|
||||
|
||||
// Wait for the update to reach the watchers.
|
||||
select {
|
||||
case <-watcher1.updateCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for update to reach watcher 1")
|
||||
}
|
||||
select {
|
||||
case <-watcher2.updateCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for update to reach watcher 2")
|
||||
}
|
||||
|
||||
// Update the listener resource on the management server to point to a new
|
||||
// route configuration resource.
|
||||
resources = e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, "new-route")},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
|
||||
// Unblock one watcher.
|
||||
onDone := <-watcher1.doneNotifierCh
|
||||
onDone()
|
||||
|
||||
// Wait for a short duration and ensure that there is no read on the stream.
|
||||
select {
|
||||
case <-adsStream.recvCh:
|
||||
t.Fatal("Recv() called on the ADS stream before all watchers have processed the previous update")
|
||||
case <-time.After(defaultTestShortTimeout):
|
||||
}
|
||||
|
||||
// Unblock the second watcher.
|
||||
onDone = <-watcher2.doneNotifierCh
|
||||
onDone()
|
||||
|
||||
// Ensure that there is a read on the stream, now that the previous update
|
||||
// has been consumed by all watchers.
|
||||
select {
|
||||
case <-adsStream.recvCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream after all watchers have processed the previous update")
|
||||
}
|
||||
|
||||
// Wait for the new update to reach the watchers.
|
||||
select {
|
||||
case <-watcher1.updateCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for update to reach watcher 1")
|
||||
}
|
||||
select {
|
||||
case <-watcher2.updateCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for update to reach watcher 2")
|
||||
}
|
||||
|
||||
// At this point, the xDS client is shut down (and the associated transport
|
||||
// is closed) without the watchers invoking their respective onDone
|
||||
// callbacks. This verifies that the closing a transport that has pending
|
||||
// watchers does not block.
|
||||
}
|
||||
|
||||
// Tests ADS stream level flow control with a multiple resources. The test does
|
||||
// the following:
|
||||
// - Starts a management server and configures two listener resources on it.
|
||||
// - Creates an xDS client to the above management server, starts a couple of
|
||||
// listener watchers for the two resources, and verifies that the update
|
||||
// reaches these watchers.
|
||||
// - These watchers don't invoke the onDone callback until explicitly
|
||||
// triggered by the test. This allows the test to verify that the next
|
||||
// Recv() call on the ADS stream does not happen until both watchers have
|
||||
// completely processed the update, i.e invoke the onDone callback.
|
||||
func (s) TestADSFlowControl_ResourceUpdates_MultipleResources(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Start an xDS management server.
|
||||
const listenerResourceName1 = "test-listener-resource-1"
|
||||
const listenerResourceName2 = "test-listener-resource-2"
|
||||
wantResourceNames := []string{listenerResourceName1, listenerResourceName2}
|
||||
requestCh := make(chan struct{}, 1)
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
|
||||
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
|
||||
if req.GetTypeUrl() != version.V3ListenerURL {
|
||||
return nil
|
||||
}
|
||||
gotResourceNames := req.GetResourceNames()
|
||||
sort.Slice(gotResourceNames, func(i, j int) bool { return req.ResourceNames[i] < req.ResourceNames[j] })
|
||||
if slices.Equal(gotResourceNames, wantResourceNames) {
|
||||
// The two resource names will be part of the initial request
|
||||
// and also the ACK. Hence, we need to make this write
|
||||
// non-blocking.
|
||||
select {
|
||||
case requestCh <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
// Create an xDS client pointing to the above server with a test transport
|
||||
// that allow monitoring the underlying stream through adsStreamCh.
|
||||
adsStreamCh := make(chan *stream, 1)
|
||||
client := createXDSClient(t, mgmtServer.Address, nodeID, &transportBuilder{adsStreamCh: adsStreamCh})
|
||||
|
||||
// Configure two watchers for two different listener resources.
|
||||
const routeConfigurationName1 = "test-route-configuration-resource-1"
|
||||
watcher1 := newBLockingListenerWatcher()
|
||||
cancel1 := client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName1, watcher1)
|
||||
defer cancel1()
|
||||
const routeConfigurationName2 = "test-route-configuration-resource-2"
|
||||
watcher2 := newBLockingListenerWatcher()
|
||||
cancel2 := client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName2, watcher2)
|
||||
defer cancel2()
|
||||
|
||||
// Wait for the wrapped ADS stream to be created.
|
||||
var adsStream *stream
|
||||
select {
|
||||
case adsStream = <-adsStreamCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for ADS stream to be created")
|
||||
}
|
||||
|
||||
// Ensure that there is a read on the stream.
|
||||
select {
|
||||
case <-adsStream.recvCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for ADS stream to be read from")
|
||||
}
|
||||
|
||||
// Wait for both resource names to be requested.
|
||||
select {
|
||||
case <-requestCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Timed out waiting for both resource names to be requested")
|
||||
}
|
||||
|
||||
// Configure the listener resources on the management server.
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{
|
||||
e2e.DefaultClientListener(listenerResourceName1, routeConfigurationName1),
|
||||
e2e.DefaultClientListener(listenerResourceName2, routeConfigurationName2),
|
||||
},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
|
||||
// At this point, we expect the management server to send both resources in
|
||||
// the same response. So, both watchers would be notified at the same time,
|
||||
// and no more Recv() calls should happen until both of them have invoked
|
||||
// their respective onDone() callbacks.
|
||||
|
||||
// The order of callback invocations among the two watchers is not
|
||||
// guaranteed. So, we select on both of them and unblock the first watcher
|
||||
// whose callback is invoked.
|
||||
var otherWatcherUpdateCh chan struct{}
|
||||
var otherWatcherDoneCh chan func()
|
||||
select {
|
||||
case <-watcher1.updateCh:
|
||||
onDone := <-watcher1.doneNotifierCh
|
||||
onDone()
|
||||
otherWatcherUpdateCh = watcher2.updateCh
|
||||
otherWatcherDoneCh = watcher2.doneNotifierCh
|
||||
case <-watcher2.updateCh:
|
||||
onDone := <-watcher2.doneNotifierCh
|
||||
onDone()
|
||||
otherWatcherUpdateCh = watcher1.updateCh
|
||||
otherWatcherDoneCh = watcher1.doneNotifierCh
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Timed out waiting for update to reach first watchers")
|
||||
}
|
||||
|
||||
// Wait for a short duration and ensure that there is no read on the stream.
|
||||
select {
|
||||
case <-adsStream.recvCh:
|
||||
t.Fatal("Recv() called on the ADS stream before all watchers have processed the previous update")
|
||||
case <-time.After(defaultTestShortTimeout):
|
||||
}
|
||||
|
||||
// Wait for the update on the second watcher and unblock it.
|
||||
select {
|
||||
case <-otherWatcherUpdateCh:
|
||||
onDone := <-otherWatcherDoneCh
|
||||
onDone()
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Timed out waiting for update to reach second watcher")
|
||||
}
|
||||
|
||||
// Ensure that there is a read on the stream, now that the previous update
|
||||
// has been consumed by all watchers.
|
||||
select {
|
||||
case <-adsStream.recvCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream after all watchers have processed the previous update")
|
||||
}
|
||||
}
|
||||
|
||||
// Test ADS stream flow control with a single resource that is expected to be
|
||||
// NACKed by the xDS client and the watcher's ResourceError() callback is
|
||||
// expected to be invoked because resource is not cached. Verifies that no
|
||||
// further reads are attempted until the error is completely processed by the
|
||||
// watcher.
|
||||
func (s) TestADSFlowControl_ResourceErrors(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Start an xDS management server.
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
|
||||
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
// Create an xDS client pointing to the above server with a test transport
|
||||
// that allow monitoring the underlying stream through adsStreamCh.
|
||||
adsStreamCh := make(chan *stream, 1)
|
||||
client := createXDSClient(t, mgmtServer.Address, nodeID, &transportBuilder{adsStreamCh: adsStreamCh})
|
||||
|
||||
// Configure a watcher for a listener resource.
|
||||
const listenerResourceName = "test-listener-resource"
|
||||
watcher := newBLockingListenerWatcher()
|
||||
cancel = client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName, watcher)
|
||||
defer cancel()
|
||||
|
||||
// Wait for the stream to be created.
|
||||
var adsStream *stream
|
||||
select {
|
||||
case adsStream = <-adsStreamCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for ADS stream to be created")
|
||||
}
|
||||
|
||||
// Configure the management server to return a single listener resource
|
||||
// which is expected to be NACKed by the client.
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{badListenerResource(t, listenerResourceName)},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
|
||||
// Ensure that there is a read on the stream.
|
||||
select {
|
||||
case <-adsStream.recvCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for ADS stream to be read from")
|
||||
}
|
||||
|
||||
// Wait for the resource error to reach the watcher.
|
||||
select {
|
||||
case <-watcher.resourceErrCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for error to reach watcher")
|
||||
}
|
||||
|
||||
// Wait for a short duration and ensure that there is no read on the stream.
|
||||
select {
|
||||
case <-adsStream.recvCh:
|
||||
t.Fatal("Recv() called on the ADS stream before all watchers have processed the previous update")
|
||||
case <-time.After(defaultTestShortTimeout):
|
||||
}
|
||||
|
||||
// Unblock one watcher.
|
||||
onDone := <-watcher.doneNotifierCh
|
||||
onDone()
|
||||
|
||||
// Ensure that there is a read on the stream, now that the previous error
|
||||
// has been consumed by the watcher.
|
||||
select {
|
||||
case <-adsStream.recvCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream after all watchers have processed the previous update")
|
||||
}
|
||||
}
|
||||
|
||||
// Test ADS stream flow control with a single resource that is deleted from the
|
||||
// management server and therefore the watcher's ResourceError()
|
||||
// callback is expected to be invoked. Verifies that no further reads are
|
||||
// attempted until the callback is completely handled by the watcher.
|
||||
func (s) TestADSFlowControl_ResourceDoesNotExist(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Start an xDS management server.
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
|
||||
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
// Create an xDS client pointing to the above server with a test transport
|
||||
// that allow monitoring the underlying stream through adsStreamCh.
|
||||
adsStreamCh := make(chan *stream, 1)
|
||||
client := createXDSClient(t, mgmtServer.Address, nodeID, &transportBuilder{adsStreamCh: adsStreamCh})
|
||||
|
||||
// Configure a watcher for a listener resource.
|
||||
const listenerResourceName = "test-listener-resource"
|
||||
const routeConfigurationName = "test-route-configuration-resource"
|
||||
watcher := newBLockingListenerWatcher()
|
||||
cancel = client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName, watcher)
|
||||
defer cancel()
|
||||
|
||||
// Wait for the ADS stream to be created.
|
||||
var adsStream *stream
|
||||
select {
|
||||
case adsStream = <-adsStreamCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for ADS stream to be created")
|
||||
}
|
||||
|
||||
// Configure the listener resource on the management server.
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
|
||||
// Ensure that there is a read on the stream.
|
||||
select {
|
||||
case <-adsStream.recvCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream")
|
||||
}
|
||||
|
||||
// Wait for the update to reach the watcher and unblock it.
|
||||
select {
|
||||
case <-watcher.updateCh:
|
||||
onDone := <-watcher.doneNotifierCh
|
||||
onDone()
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for update to reach watcher 1")
|
||||
}
|
||||
|
||||
// Ensure that there is a read on the stream.
|
||||
select {
|
||||
case <-adsStream.recvCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream")
|
||||
}
|
||||
|
||||
// Remove the listener resource on the management server.
|
||||
resources = e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
|
||||
// Wait for the resource not found callback to be invoked.
|
||||
select {
|
||||
case <-watcher.resourceErrCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for resource not found callback to be invoked on the watcher")
|
||||
}
|
||||
|
||||
// Wait for a short duration and ensure that there is no read on the stream.
|
||||
select {
|
||||
case <-adsStream.recvCh:
|
||||
t.Fatal("Recv() called on the ADS stream before all watchers have processed the previous update")
|
||||
case <-time.After(defaultTestShortTimeout):
|
||||
}
|
||||
|
||||
// Unblock the watcher.
|
||||
onDone := <-watcher.doneNotifierCh
|
||||
onDone()
|
||||
|
||||
// Ensure that there is a read on the stream.
|
||||
select {
|
||||
case <-adsStream.recvCh:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream")
|
||||
}
|
||||
}
|
||||
202
internal/xds/clients/xdsclient/test/ads_stream_restart_test.go
Normal file
202
internal/xds/clients/xdsclient/test/ads_stream_restart_test.go
Normal file
@@ -0,0 +1,202 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2024 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/internal/testutils/xds/e2e"
|
||||
"google.golang.org/grpc/internal/xds/clients/grpctransport"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
|
||||
|
||||
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
|
||||
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
|
||||
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
|
||||
)
|
||||
|
||||
// Tests that an ADS stream is restarted after a connection failure. Also
|
||||
// verifies that if there were any watches registered before the connection
|
||||
// failed, those resources are re-requested after the stream is restarted.
|
||||
func (s) TestADS_ResourcesAreRequestedAfterStreamRestart(t *testing.T) {
|
||||
// Create a restartable listener that can simulate a broken ADS stream.
|
||||
l, err := net.Listen("tcp", "localhost:0")
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen() failed: %v", err)
|
||||
}
|
||||
lis := testutils.NewRestartableListener(l)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Start an xDS management server that uses a couple of channels to inform
|
||||
// the test about the specific LDS and CDS resource names being requested.
|
||||
ldsResourcesCh := make(chan []string, 2)
|
||||
streamOpened := make(chan struct{}, 1)
|
||||
streamClosed := make(chan struct{}, 1)
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
|
||||
Listener: lis,
|
||||
OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
|
||||
t.Logf("Received request for resources: %v of type %s", req.GetResourceNames(), req.GetTypeUrl())
|
||||
|
||||
// Drain the resource name channels before writing to them to ensure
|
||||
// that the most recently requested names are made available to the
|
||||
// test.
|
||||
if req.GetTypeUrl() == version.V3ListenerURL {
|
||||
select {
|
||||
case <-ldsResourcesCh:
|
||||
default:
|
||||
}
|
||||
ldsResourcesCh <- req.GetResourceNames()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
OnStreamClosed: func(int64, *v3corepb.Node) {
|
||||
select {
|
||||
case streamClosed <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
|
||||
},
|
||||
OnStreamOpen: func(context.Context, int64, string) error {
|
||||
select {
|
||||
case streamOpened <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
// Create a listener resource on the management server.
|
||||
const listenerName = "listener"
|
||||
const routeConfigName = "route-config"
|
||||
nodeID := uuid.New().String()
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeConfigName)},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create an xDS client pointing to the above server.
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
|
||||
|
||||
// Register a watch for a listener resource.
|
||||
lw := newListenerWatcher()
|
||||
ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
|
||||
defer ldsCancel()
|
||||
|
||||
// Verify that an ADS stream is opened and an LDS request with the above
|
||||
// resource name is sent.
|
||||
select {
|
||||
case <-streamOpened:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Timeout when waiting for ADS stream to open")
|
||||
}
|
||||
if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify the update received by the watcher.
|
||||
wantListenerUpdate := listenerUpdateErrTuple{
|
||||
update: listenerUpdate{
|
||||
RouteConfigName: routeConfigName,
|
||||
},
|
||||
}
|
||||
if err := verifyListenerUpdate(ctx, lw.updateCh, wantListenerUpdate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create another listener resource on the management server, in addition
|
||||
// to the existing listener resource.
|
||||
const listenerName2 = "listener2"
|
||||
const routeConfigName2 = "route-config2"
|
||||
resources = e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeConfigName), e2e.DefaultClientListener(listenerName2, routeConfigName2)},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Register a watch for another listener resource, and verify that a LDS request
|
||||
// with the both listener resource names are sent.
|
||||
lw2 := newListenerWatcher()
|
||||
ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, listenerName2, lw2)
|
||||
if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName, listenerName2}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify the update received by the watcher.
|
||||
wantListenerUpdate = listenerUpdateErrTuple{
|
||||
update: listenerUpdate{
|
||||
RouteConfigName: routeConfigName2,
|
||||
},
|
||||
}
|
||||
if err := verifyListenerUpdate(ctx, lw2.updateCh, wantListenerUpdate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Cancel the watch for the second listener resource, and verify that an LDS
|
||||
// request with only first listener resource names is sent.
|
||||
ldsCancel2()
|
||||
if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Stop the restartable listener and wait for the stream to close.
|
||||
lis.Stop()
|
||||
select {
|
||||
case <-streamClosed:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Timeout when waiting for ADS stream to close")
|
||||
}
|
||||
|
||||
// Restart the restartable listener and wait for the stream to open.
|
||||
lis.Restart()
|
||||
select {
|
||||
case <-streamOpened:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Timeout when waiting for ADS stream to open")
|
||||
}
|
||||
|
||||
// Verify that the first listener resource is requested again.
|
||||
if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Wait for a short duration and verify that no LDS request is sent, since
|
||||
// there are no resources being watched.
|
||||
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
||||
defer sCancel()
|
||||
select {
|
||||
case <-sCtx.Done():
|
||||
case names := <-ldsResourcesCh:
|
||||
t.Fatalf("LDS request sent for resource names %v, when expecting no request", names)
|
||||
}
|
||||
}
|
||||
222
internal/xds/clients/xdsclient/test/ads_stream_watch_test.go
Normal file
222
internal/xds/clients/xdsclient/test/ads_stream_watch_test.go
Normal file
@@ -0,0 +1,222 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2024 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/internal/testutils/xds/e2e"
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
"google.golang.org/grpc/internal/xds/clients/grpctransport"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient"
|
||||
xdsclientinternal "google.golang.org/grpc/internal/xds/clients/xdsclient/internal"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
|
||||
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
|
||||
)
|
||||
|
||||
func waitForResourceWatchState(ctx context.Context, client *xdsclient.XDSClient, resourceName string, wantState xdsresource.WatchState, wantTimer bool) error {
|
||||
var lastErr error
|
||||
for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
|
||||
err := verifyResourceWatchState(client, resourceName, wantState, wantTimer)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return fmt.Errorf("timeout when waiting for expected watch state for resource %q: %v", resourceName, lastErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyResourceWatchState(client *xdsclient.XDSClient, resourceName string, wantState xdsresource.WatchState, wantTimer bool) error {
|
||||
resourceWatchStateForTesting := xdsclientinternal.ResourceWatchStateForTesting.(func(*xdsclient.XDSClient, xdsclient.ResourceType, string) (xdsresource.ResourceWatchState, error))
|
||||
gotState, err := resourceWatchStateForTesting(client, listenerType, resourceName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get watch state for resource %q: %v", resourceName, err)
|
||||
}
|
||||
if gotState.State != wantState {
|
||||
return fmt.Errorf("watch state for resource %q is %v, want %v", resourceName, gotState.State, wantState)
|
||||
}
|
||||
if (gotState.ExpiryTimer != nil) != wantTimer {
|
||||
return fmt.Errorf("expiry timer for resource %q is %t, want %t", resourceName, gotState.ExpiryTimer != nil, wantTimer)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Tests the state transitions of the resource specific watch state within the
|
||||
// ADS stream, specifically when the stream breaks (for both resources that have
|
||||
// been previously received and for resources that are yet to be received).
|
||||
func (s) TestADS_WatchState_StreamBreaks(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Create an xDS management server with a restartable listener.
|
||||
l, err := net.Listen("tcp", "localhost:0")
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen() failed: %v", err)
|
||||
}
|
||||
lis := testutils.NewRestartableListener(l)
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lis})
|
||||
|
||||
// Create an xDS client pointing to the above server.
|
||||
nodeID := uuid.New().String()
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
|
||||
|
||||
// Create a watch for the first listener resource and verify that the timer
|
||||
// is running and the watch state is `requested`.
|
||||
const listenerName1 = "listener1"
|
||||
ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, listenerName1, noopListenerWatcher{})
|
||||
defer ldsCancel1()
|
||||
if err := waitForResourceWatchState(ctx, client, listenerName1, xdsresource.ResourceWatchStateRequested, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Configure the first resource on the management server. This should result
|
||||
// in the resource being pushed to the xDS client and should result in the
|
||||
// timer getting stopped and the watch state moving to `received`.
|
||||
const routeConfigName = "route-config"
|
||||
listenerResource1 := e2e.DefaultClientListener(listenerName1, routeConfigName)
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{listenerResource1},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := waitForResourceWatchState(ctx, client, listenerName1, xdsresource.ResourceWatchStateReceived, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create a watch for the second listener resource and verify that the timer
|
||||
// is running and the watch state is `requested`.
|
||||
const listenerName2 = "listener2"
|
||||
ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, listenerName2, noopListenerWatcher{})
|
||||
defer ldsCancel2()
|
||||
if err := waitForResourceWatchState(ctx, client, listenerName2, xdsresource.ResourceWatchStateRequested, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Stop the server to break the ADS stream. Since the first resource was
|
||||
// already received, this should not change anything for it. But for the
|
||||
// second resource, it should result in the timer getting stopped and the
|
||||
// watch state moving to `started`.
|
||||
lis.Stop()
|
||||
if err := waitForResourceWatchState(ctx, client, listenerName2, xdsresource.ResourceWatchStateStarted, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifyResourceWatchState(client, listenerName1, xdsresource.ResourceWatchStateReceived, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Restart the server and verify that the timer is running and the watch
|
||||
// state is `requested`, for the second resource. For the first resource,
|
||||
// nothing should change.
|
||||
lis.Restart()
|
||||
if err := waitForResourceWatchState(ctx, client, listenerName2, xdsresource.ResourceWatchStateRequested, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifyResourceWatchState(client, listenerName1, xdsresource.ResourceWatchStateReceived, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Configure the second resource on the management server. This should result
|
||||
// in the resource being pushed to the xDS client and should result in the
|
||||
// timer getting stopped and the watch state moving to `received`.
|
||||
listenerResource2 := e2e.DefaultClientListener(listenerName2, routeConfigName)
|
||||
resources = e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{listenerResource1, listenerResource2},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := waitForResourceWatchState(ctx, client, listenerName2, xdsresource.ResourceWatchStateReceived, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests the behavior of the xDS client when a resource watch timer expires and
|
||||
// verifies the resource watch state transitions as expected.
|
||||
func (s) TestADS_WatchState_TimerFires(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Start an xDS management server.
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
|
||||
|
||||
// Create an xDS client with bootstrap pointing to the above server, and a
|
||||
// short resource expiry timeout.
|
||||
nodeID := uuid.New().String()
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
|
||||
si := clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}},
|
||||
Node: clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
ResourceTypes: resourceTypes,
|
||||
// Xdstp resource names used in this test do not specify an
|
||||
// authority. These will end up looking up an entry with the
|
||||
// empty key in the authorities map. Having an entry with an
|
||||
// empty key and empty configuration, results in these
|
||||
// resources also using the top-level configuration.
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
"": {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
WatchExpiryTimeout: defaultTestWatchExpiryTimeout,
|
||||
}
|
||||
|
||||
// Create an xDS client with the above config.
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { client.Close() })
|
||||
// Create a watch for the first listener resource and verify that the timer
|
||||
// is running and the watch state is `requested`.
|
||||
const listenerName = "listener"
|
||||
ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, listenerName, noopListenerWatcher{})
|
||||
defer ldsCancel1()
|
||||
if err := waitForResourceWatchState(ctx, client, listenerName, xdsresource.ResourceWatchStateRequested, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Since the resource is not configured on the management server, the watch
|
||||
// expiry timer is expected to fire, and the watch state should move to
|
||||
// `timeout`.
|
||||
if err := waitForResourceWatchState(ctx, client, listenerName, xdsresource.ResourceWatchStateTimeout, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
334
internal/xds/clients/xdsclient/test/authority_test.go
Normal file
334
internal/xds/clients/xdsclient/test/authority_test.go
Normal file
@@ -0,0 +1,334 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2022 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
"google.golang.org/grpc/internal/xds/clients/grpctransport"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
|
||||
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
|
||||
)
|
||||
|
||||
const (
|
||||
testAuthority1 = "test-authority1"
|
||||
testAuthority2 = "test-authority2"
|
||||
testAuthority3 = "test-authority3"
|
||||
)
|
||||
|
||||
var (
|
||||
// These two resources use `testAuthority1`, which contains an empty server
|
||||
// config and therefore will use the default management server.
|
||||
authorityTestResourceName11 = buildResourceName(listenerResourceTypeName, testAuthority1, ldsName, nil)
|
||||
authorityTestResourceName12 = buildResourceName(listenerResourceTypeName, testAuthority1, ldsName+"2", nil)
|
||||
// This resource uses `testAuthority2`, which contains an empty server
|
||||
// config and therefore will use the default management server.
|
||||
authorityTestResourceName2 = buildResourceName(listenerResourceTypeName, testAuthority2, ldsName+"3", nil)
|
||||
// This resource uses `testAuthority3`, which contains a non-empty server
|
||||
// config, and therefore will use the non-default management server.
|
||||
authorityTestResourceName3 = buildResourceName(listenerResourceTypeName, testAuthority3, ldsName+"3", nil)
|
||||
)
|
||||
|
||||
// setupForAuthorityTests spins up two management servers, one to act as the
|
||||
// default and the other to act as the non-default. It also creates a
|
||||
// xDS client configuration with three authorities (the first two pointing to
|
||||
// the default and the third one pointing to the non-default).
|
||||
//
|
||||
// Returns two listeners used by the default and non-default management servers
|
||||
// respectively, and the xDS client.
|
||||
func setupForAuthorityTests(ctx context.Context, t *testing.T) (*testutils.ListenerWrapper, *testutils.ListenerWrapper, *xdsclient.XDSClient) {
|
||||
// Create listener wrappers which notify on to a channel whenever a new
|
||||
// connection is accepted. We use this to track the number of transports
|
||||
// used by the xDS client.
|
||||
lisDefault := testutils.NewListenerWrapper(t, nil)
|
||||
lisNonDefault := testutils.NewListenerWrapper(t, nil)
|
||||
|
||||
// Start a management server to act as the default authority.
|
||||
defaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisDefault})
|
||||
|
||||
// Start a management server to act as the non-default authority.
|
||||
nonDefaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisNonDefault})
|
||||
|
||||
// Create a bootstrap configuration with two non-default authorities which
|
||||
// have empty server configs, and therefore end up using the default server
|
||||
// config, which points to the above management server.
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
|
||||
ext := grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}
|
||||
siDefault := clients.ServerIdentifier{
|
||||
ServerURI: defaultAuthorityServer.Address,
|
||||
Extensions: ext,
|
||||
}
|
||||
siNonDefault := clients.ServerIdentifier{
|
||||
ServerURI: nonDefaultAuthorityServer.Address,
|
||||
Extensions: ext,
|
||||
}
|
||||
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{ServerIdentifier: siDefault}},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
ResourceTypes: resourceTypes,
|
||||
// Xdstp style resource names used in this test use a slash removed
|
||||
// version of t.Name as their authority, and the empty config
|
||||
// results in the top-level xds server configuration being used for
|
||||
// this authority.
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
testAuthority1: {XDSServers: []xdsclient.ServerConfig{}},
|
||||
testAuthority2: {XDSServers: []xdsclient.ServerConfig{}},
|
||||
testAuthority3: {XDSServers: []xdsclient.ServerConfig{{ServerIdentifier: siNonDefault}}},
|
||||
},
|
||||
WatchExpiryTimeout: defaultTestWatchExpiryTimeout,
|
||||
}
|
||||
|
||||
// Create an xDS client with the above config.
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{
|
||||
e2e.DefaultClientListener(authorityTestResourceName11, rdsName),
|
||||
e2e.DefaultClientListener(authorityTestResourceName12, rdsName),
|
||||
e2e.DefaultClientListener(authorityTestResourceName2, rdsName),
|
||||
e2e.DefaultClientListener(authorityTestResourceName3, rdsName),
|
||||
},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := defaultAuthorityServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
return lisDefault, lisNonDefault, client
|
||||
}
|
||||
|
||||
// Tests the xdsChannel sharing logic among authorities. The test verifies the
|
||||
// following scenarios:
|
||||
// - A watch for a resource name with an authority matching an existing watch
|
||||
// should not result in a new transport being created.
|
||||
// - A watch for a resource name with different authority name but same
|
||||
// authority config as an existing watch should not result in a new transport
|
||||
// being created.
|
||||
func (s) TestAuthority_XDSChannelSharing(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
lis, _, client := setupForAuthorityTests(ctx, t)
|
||||
defer client.Close()
|
||||
|
||||
// Verify that no connection is established to the management server at this
|
||||
// point. A transport is created only when a resource (which belongs to that
|
||||
// authority) is requested.
|
||||
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
||||
defer sCancel()
|
||||
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
|
||||
t.Fatal("Unexpected new transport created to management server")
|
||||
}
|
||||
|
||||
// Request the first resource. Verify that a new transport is created.
|
||||
watcher := noopListenerWatcher{}
|
||||
ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName11, watcher)
|
||||
defer ldsCancel1()
|
||||
if _, err := lis.NewConnCh.Receive(ctx); err != nil {
|
||||
t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
|
||||
}
|
||||
|
||||
// Request the second resource. Verify that no new transport is created.
|
||||
ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName12, watcher)
|
||||
defer ldsCancel2()
|
||||
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
|
||||
defer sCancel()
|
||||
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
|
||||
t.Fatal("Unexpected new transport created to management server")
|
||||
}
|
||||
|
||||
// Request the third resource. Verify that no new transport is created.
|
||||
ldsCancel3 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName2, watcher)
|
||||
defer ldsCancel3()
|
||||
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
|
||||
defer sCancel()
|
||||
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
|
||||
t.Fatal("Unexpected new transport created to management server")
|
||||
}
|
||||
}
|
||||
|
||||
// Test the xdsChannel close logic. The test verifies that the xDS client
|
||||
// closes an xdsChannel immediately after the last watch is canceled.
|
||||
func (s) TestAuthority_XDSChannelClose(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
lis, _, client := setupForAuthorityTests(ctx, t)
|
||||
defer client.Close()
|
||||
|
||||
// Request the first resource. Verify that a new transport is created.
|
||||
watcher := noopListenerWatcher{}
|
||||
ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName11, watcher)
|
||||
val, err := lis.NewConnCh.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
|
||||
}
|
||||
conn := val.(*testutils.ConnWrapper)
|
||||
|
||||
// Request the second resource. Verify that no new transport is created.
|
||||
ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName12, watcher)
|
||||
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
||||
defer sCancel()
|
||||
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
|
||||
t.Fatal("Unexpected new transport created to management server")
|
||||
}
|
||||
|
||||
// Cancel both watches, and verify that the connection to the management
|
||||
// server is closed.
|
||||
ldsCancel1()
|
||||
ldsCancel2()
|
||||
if _, err := conn.CloseCh.Receive(ctx); err != nil {
|
||||
t.Fatal("Timeout when waiting for connection to management server to be closed")
|
||||
}
|
||||
}
|
||||
|
||||
// Tests the scenario where the primary management server is unavailable at
|
||||
// startup and the xDS client falls back to the secondary. The test verifies
|
||||
// that the resource watcher is not notified of the connectivity failure until
|
||||
// all servers have failed.
|
||||
func (s) TestAuthority_Fallback(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Create primary and secondary management servers with restartable
|
||||
// listeners.
|
||||
l, err := net.Listen("tcp", "localhost:0")
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen() failed: %v", err)
|
||||
}
|
||||
primaryLis := testutils.NewRestartableListener(l)
|
||||
primaryMgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis})
|
||||
l, err = net.Listen("tcp", "localhost:0")
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen() failed: %v", err)
|
||||
}
|
||||
secondaryLis := testutils.NewRestartableListener(l)
|
||||
secondaryMgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: secondaryLis})
|
||||
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
|
||||
psi := clients.ServerIdentifier{
|
||||
ServerURI: primaryMgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
ssi := clients.ServerIdentifier{
|
||||
ServerURI: secondaryMgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
|
||||
// Create config with the above primary and fallback management servers,
|
||||
// and an xDS client with that configuration.
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{ServerIdentifier: psi}, {ServerIdentifier: ssi}},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
ResourceTypes: resourceTypes,
|
||||
// Xdstp resource names used in this test do not specify an
|
||||
// authority. These will end up looking up an entry with the
|
||||
// empty key in the authorities map. Having an entry with an
|
||||
// empty key and empty configuration, results in these
|
||||
// resources also using the top-level configuration.
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
"": {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
}
|
||||
|
||||
// Create an xDS client with the above config.
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
const listenerName = "listener"
|
||||
const rdsPrimaryName = "rds-primary"
|
||||
const rdsSecondaryName = "rds-secondary"
|
||||
|
||||
// Create a Cluster resource on the primary.
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, rdsPrimaryName)},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := primaryMgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update primary management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
|
||||
// Create a Cluster resource on the secondary .
|
||||
resources = e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, rdsSecondaryName)},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := secondaryMgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update primary management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
|
||||
// Stop the primary.
|
||||
primaryLis.Close()
|
||||
|
||||
// Register a watch.
|
||||
watcher := newListenerWatcher()
|
||||
ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher)
|
||||
defer ldsCancel()
|
||||
|
||||
// Ensure that the connectivity error callback is not called. Since, this
|
||||
// is the first watch without cached resource, it checks for resourceErrCh
|
||||
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
||||
defer sCancel()
|
||||
if v, err := watcher.resourceErrCh.Receive(sCtx); err != context.DeadlineExceeded {
|
||||
t.Fatalf("Resource error callback on the watcher with error: %v", v.(error))
|
||||
}
|
||||
|
||||
// Ensure that the resource update callback is invoked.
|
||||
wantUpdate := listenerUpdateErrTuple{
|
||||
update: listenerUpdate{
|
||||
RouteConfigName: rdsSecondaryName,
|
||||
},
|
||||
}
|
||||
if err := verifyListenerUpdate(ctx, watcher.updateCh, wantUpdate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Stop the secondary.
|
||||
secondaryLis.Close()
|
||||
|
||||
// Ensure that the connectivity error callback is called as ambient error
|
||||
// since cached resource exist.
|
||||
if _, err := watcher.ambientErrCh.Receive(ctx); err != nil {
|
||||
t.Fatal("Timeout when waiting for ambient error callback on the watcher")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2026 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
"google.golang.org/grpc/internal/xds/clients/grpctransport"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils/fakeserver"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
"google.golang.org/protobuf/types/known/wrapperspb"
|
||||
|
||||
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
|
||||
)
|
||||
|
||||
// customTestResourceType is a fake resource type used for testing.
|
||||
var customTestResourceType = xdsclient.ResourceType{
|
||||
TypeURL: "type.googleapis.com/google.protobuf.StringValue",
|
||||
TypeName: "CustomResource",
|
||||
AllResourcesRequiredInSotW: false,
|
||||
Decoder: customTestDecoder{},
|
||||
}
|
||||
|
||||
type customTestDecoder struct{}
|
||||
|
||||
func (customTestDecoder) Decode(resource *xdsclient.AnyProto, _ xdsclient.DecodeOptions) (*xdsclient.DecodeResult, error) {
|
||||
any := resource.ToAny()
|
||||
if any.GetTypeUrl() != customTestResourceType.TypeURL {
|
||||
return nil, fmt.Errorf("unexpected resource type: %q", any.GetTypeUrl())
|
||||
}
|
||||
var val wrapperspb.StringValue
|
||||
if err := proto.Unmarshal(any.GetValue(), &val); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal resource: %v", err)
|
||||
}
|
||||
|
||||
// We expect the value to be "name|content". This allows us to extract the
|
||||
// name which is required by the xDS client to match the watch.
|
||||
str := val.GetValue()
|
||||
name, content, found := strings.Cut(str, "|")
|
||||
if !found {
|
||||
return nil, fmt.Errorf("invalid format: expected 'name|content', got %q", str)
|
||||
}
|
||||
|
||||
return &xdsclient.DecodeResult{
|
||||
Name: name,
|
||||
Resource: &customTestResourceData{val: content},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type customTestResourceData struct {
|
||||
val string
|
||||
}
|
||||
|
||||
func (c *customTestResourceData) Equal(other xdsclient.ResourceData) bool {
|
||||
if c == nil && other == nil {
|
||||
return true
|
||||
}
|
||||
if c == nil || other == nil {
|
||||
return false
|
||||
}
|
||||
o, ok := other.(*customTestResourceData)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return c.val == o.val
|
||||
}
|
||||
|
||||
func (c *customTestResourceData) Bytes() []byte {
|
||||
return []byte(c.val)
|
||||
}
|
||||
|
||||
// customTestWatcher is a watcher for the custom resource type.
|
||||
type customTestWatcher struct {
|
||||
updateCh chan xdsclient.ResourceData
|
||||
errCh chan error
|
||||
}
|
||||
|
||||
func newCustomTestWatcher() *customTestWatcher {
|
||||
return &customTestWatcher{
|
||||
updateCh: make(chan xdsclient.ResourceData, 1),
|
||||
errCh: make(chan error, 1),
|
||||
}
|
||||
}
|
||||
|
||||
func (w *customTestWatcher) ResourceChanged(update xdsclient.ResourceData, onDone func()) {
|
||||
w.updateCh <- update
|
||||
onDone()
|
||||
}
|
||||
|
||||
func (w *customTestWatcher) ResourceError(err error, onDone func()) {
|
||||
w.errCh <- fmt.Errorf("ResourceError: %v", err)
|
||||
onDone()
|
||||
}
|
||||
|
||||
func (w *customTestWatcher) AmbientError(err error, onDone func()) {
|
||||
w.errCh <- fmt.Errorf("AmbientError: %v", err)
|
||||
onDone()
|
||||
}
|
||||
|
||||
// Tests that the xDS client can watch a custom resource type that is injected
|
||||
// via the config.
|
||||
func (s) TestCustomResourceWatch(t *testing.T) {
|
||||
const authority = "my-authority"
|
||||
resourceName := "xdstp://" + authority + "/" + customTestResourceType.TypeURL + "/my-resource"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
resourceValue string
|
||||
wantUpdate string
|
||||
wantNACK string
|
||||
}{
|
||||
{
|
||||
name: "valid_resource",
|
||||
resourceValue: resourceName + "|hello world",
|
||||
wantUpdate: "hello world",
|
||||
},
|
||||
{
|
||||
name: "decode_error",
|
||||
resourceValue: "malformed-value",
|
||||
wantNACK: "invalid format: expected 'name|content'",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
// Start a fake xDS management server.
|
||||
mgmtServer, cleanup, err := fakeserver.StartServer(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to start fake xDS server: %v", err)
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
resourceTypes := map[string]xdsclient.ResourceType{
|
||||
customTestResourceType.TypeURL: customTestResourceType,
|
||||
}
|
||||
si := clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
nodeID := uuid.New().String()
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
ResourceTypes: resourceTypes,
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
authority: {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
}
|
||||
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
watcher := newCustomTestWatcher()
|
||||
cancelWatch := client.WatchResource(customTestResourceType.TypeURL, resourceName, watcher)
|
||||
defer cancelWatch()
|
||||
|
||||
// Wait for the discovery request.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Verify the request type URL.
|
||||
v, err := mgmtServer.XDSRequestChan.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Timeout when waiting for a DiscoveryRequest message: %v", err)
|
||||
}
|
||||
req := v.(*fakeserver.Request).Req.(*v3discoverypb.DiscoveryRequest)
|
||||
if req.GetTypeUrl() != customTestResourceType.TypeURL {
|
||||
t.Fatalf("DiscoveryRequest TypeUrl = %v, want %v", req.GetTypeUrl(), customTestResourceType.TypeURL)
|
||||
}
|
||||
if len(req.GetResourceNames()) != 1 || req.GetResourceNames()[0] != resourceName {
|
||||
t.Fatalf("DiscoveryRequest ResourceNames = %v, want [%v]", req.GetResourceNames(), resourceName)
|
||||
}
|
||||
|
||||
// Send a response with the custom resource.
|
||||
resource, err := anypb.New(&wrapperspb.StringValue{Value: test.resourceValue})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to marshal resource: %v", err)
|
||||
}
|
||||
mgmtServer.XDSResponseChan <- &fakeserver.Response{
|
||||
Resp: &v3discoverypb.DiscoveryResponse{
|
||||
TypeUrl: customTestResourceType.TypeURL,
|
||||
VersionInfo: "1",
|
||||
Resources: []*anypb.Any{resource},
|
||||
},
|
||||
}
|
||||
|
||||
if test.wantNACK != "" {
|
||||
// Verify the NACK.
|
||||
// We expect a new DiscoveryRequest with ErrorDetail set.
|
||||
v, err = mgmtServer.XDSRequestChan.Receive(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Timeout when waiting for NACK DiscoveryRequest message: %v", err)
|
||||
}
|
||||
req = v.(*fakeserver.Request).Req.(*v3discoverypb.DiscoveryRequest)
|
||||
if req.GetTypeUrl() != customTestResourceType.TypeURL {
|
||||
t.Fatalf("DiscoveryRequest TypeUrl = %v, want %v", req.GetTypeUrl(), customTestResourceType.TypeURL)
|
||||
}
|
||||
if req.GetErrorDetail() == nil {
|
||||
t.Fatalf("DiscoveryRequest ErrorDetail is nil, want non-nil")
|
||||
}
|
||||
if !strings.Contains(req.GetErrorDetail().GetMessage(), test.wantNACK) {
|
||||
t.Fatalf("DiscoveryRequest ErrorDetail = %v, want substring %q", req.GetErrorDetail().GetMessage(), test.wantNACK)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Verify the update.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timeout waiting for resource update")
|
||||
case err := <-watcher.errCh:
|
||||
t.Fatalf("Received unexpected error: %v", err)
|
||||
case got := <-watcher.updateCh:
|
||||
gotData, ok := got.(*customTestResourceData)
|
||||
if !ok {
|
||||
t.Fatalf("Received unexpected data type: %T", got)
|
||||
}
|
||||
if gotData.val != test.wantUpdate {
|
||||
t.Fatalf("Received resource value %q, want %q", gotData.val, test.wantUpdate)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
557
internal/xds/clients/xdsclient/test/dump_test.go
Normal file
557
internal/xds/clients/xdsclient/test/dump_test.go
Normal file
@@ -0,0 +1,557 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2022 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/uuid"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
"google.golang.org/grpc/internal/xds/clients/grpctransport"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/pretty"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/testing/protocmp"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
|
||||
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
|
||||
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
|
||||
v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
|
||||
v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
|
||||
)
|
||||
|
||||
func makeGenericXdsConfig(typeURL, name, version string, status v3adminpb.ClientResourceStatus, config *anypb.Any, failure *v3adminpb.UpdateFailureState) *v3statuspb.ClientConfig_GenericXdsConfig {
|
||||
return &v3statuspb.ClientConfig_GenericXdsConfig{
|
||||
TypeUrl: typeURL,
|
||||
Name: name,
|
||||
VersionInfo: version,
|
||||
ClientStatus: status,
|
||||
XdsConfig: config,
|
||||
ErrorState: failure,
|
||||
}
|
||||
}
|
||||
|
||||
func checkResourceDump(ctx context.Context, want *v3statuspb.ClientStatusResponse, client *xdsclient.XDSClient) error {
|
||||
var cmpOpts = cmp.Options{
|
||||
protocmp.Transform(),
|
||||
protocmp.IgnoreFields((*v3statuspb.ClientConfig_GenericXdsConfig)(nil), "last_updated"),
|
||||
protocmp.IgnoreFields((*v3statuspb.ClientConfig)(nil), "client_scope"),
|
||||
protocmp.IgnoreFields((*v3adminpb.UpdateFailureState)(nil), "last_update_attempt", "details"),
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
|
||||
b, err := client.DumpResources()
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
got := &v3statuspb.ClientStatusResponse{}
|
||||
if err := proto.Unmarshal(b, got); err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
// Sort the client configs based on the `client_scope` field.
|
||||
slices.SortFunc(got.GetConfig(), func(a, b *v3statuspb.ClientConfig) int {
|
||||
return strings.Compare(a.ClientScope, b.ClientScope)
|
||||
})
|
||||
// Sort the resource configs based on the type_url and name fields.
|
||||
for _, cc := range got.GetConfig() {
|
||||
slices.SortFunc(cc.GetGenericXdsConfigs(), func(a, b *v3statuspb.ClientConfig_GenericXdsConfig) int {
|
||||
if strings.Compare(a.TypeUrl, b.TypeUrl) == 0 {
|
||||
return strings.Compare(a.Name, b.Name)
|
||||
}
|
||||
return strings.Compare(a.TypeUrl, b.TypeUrl)
|
||||
})
|
||||
}
|
||||
diff := cmp.Diff(want, got, cmpOpts)
|
||||
if diff == "" {
|
||||
return nil
|
||||
}
|
||||
lastErr = fmt.Errorf("received unexpected resource dump, diff (-got, +want):\n%s, got: %s\n want:%s", diff, pretty.ToJSON(got), pretty.ToJSON(want))
|
||||
}
|
||||
return fmt.Errorf("timeout when waiting for resource dump to reach expected state: %v", lastErr)
|
||||
}
|
||||
|
||||
// Tests the scenario where there are multiple xDS clients talking to the same
|
||||
// management server, and requesting the same set of resources. Verifies that
|
||||
// under all circumstances, both xDS clients receive the same configuration from
|
||||
// the server.
|
||||
func (s) TestDumpResources_ManyToOne(t *testing.T) {
|
||||
// Initialize the xDS resources to be used in this test.
|
||||
ldsTargets := []string{"lds.target.good:0000", "lds.target.good:1111"}
|
||||
rdsTargets := []string{"route-config-0", "route-config-1"}
|
||||
listeners := make([]*v3listenerpb.Listener, len(ldsTargets))
|
||||
listenerAnys := make([]*anypb.Any, len(ldsTargets))
|
||||
for i := range ldsTargets {
|
||||
listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i])
|
||||
listenerAnys[i] = testutils.MarshalAny(t, listeners[i])
|
||||
}
|
||||
|
||||
// Spin up an xDS management server on a local port.
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
|
||||
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
|
||||
si := clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}},
|
||||
Node: clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
ResourceTypes: resourceTypes,
|
||||
// Xdstp resource names used in this test do not specify an
|
||||
// authority. These will end up looking up an entry with the
|
||||
// empty key in the authorities map. Having an entry with an
|
||||
// empty key and empty configuration, results in these
|
||||
// resources also using the top-level configuration.
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
"": {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
}
|
||||
|
||||
// Create two xDS clients with the above config.
|
||||
client1, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client1.Close()
|
||||
client2, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client2.Close()
|
||||
|
||||
// Dump resources and expect empty configs.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
wantNode := &v3corepb.Node{
|
||||
Id: nodeID,
|
||||
UserAgentName: "user-agent",
|
||||
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
|
||||
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
|
||||
}
|
||||
wantResp := &v3statuspb.ClientStatusResponse{
|
||||
Config: []*v3statuspb.ClientConfig{
|
||||
{
|
||||
Node: wantNode,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Register watches, dump resources and expect configs in requested state.
|
||||
for _, xdsC := range []*xdsclient.XDSClient{client1, client2} {
|
||||
for _, target := range ldsTargets {
|
||||
xdsC.WatchResource(xdsresource.V3ListenerURL, target, noopListenerWatcher{})
|
||||
}
|
||||
}
|
||||
wantConfigs := []*v3statuspb.ClientConfig_GenericXdsConfig{
|
||||
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
|
||||
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
|
||||
}
|
||||
wantResp = &v3statuspb.ClientStatusResponse{
|
||||
Config: []*v3statuspb.ClientConfig{
|
||||
{
|
||||
Node: wantNode,
|
||||
GenericXdsConfigs: wantConfigs,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Configure the resources on the management server.
|
||||
if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: listeners,
|
||||
SkipValidation: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Dump resources and expect ACK configs.
|
||||
wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{
|
||||
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[0], nil),
|
||||
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil),
|
||||
}
|
||||
wantResp = &v3statuspb.ClientStatusResponse{
|
||||
Config: []*v3statuspb.ClientConfig{
|
||||
{
|
||||
Node: wantNode,
|
||||
GenericXdsConfigs: wantConfigs,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Update the first resource of each type in the management server to a
|
||||
// value which is expected to be NACK'ed by the xDS client.
|
||||
listeners[0] = func() *v3listenerpb.Listener {
|
||||
hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{})
|
||||
return &v3listenerpb.Listener{
|
||||
Name: ldsTargets[0],
|
||||
ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm},
|
||||
}
|
||||
}()
|
||||
if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: listeners,
|
||||
SkipValidation: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{
|
||||
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_NACKED, listenerAnys[0], &v3adminpb.UpdateFailureState{VersionInfo: "2"}),
|
||||
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "2", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil),
|
||||
}
|
||||
wantResp = &v3statuspb.ClientStatusResponse{
|
||||
Config: []*v3statuspb.ClientConfig{
|
||||
{
|
||||
Node: wantNode,
|
||||
GenericXdsConfigs: wantConfigs,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests the scenario where there are multiple xDS client talking to different
|
||||
// management server, and requesting different set of resources.
|
||||
func (s) TestDumpResources_ManyToMany(t *testing.T) {
|
||||
// Initialize the xDS resources to be used in this test:
|
||||
// - The first xDS client watches old style resource names, and thereby
|
||||
// requests these resources from the top-level xDS server.
|
||||
// - The second xDS client watches new style resource names with a non-empty
|
||||
// authority, and thereby requests these resources from the server
|
||||
// configuration for that authority.
|
||||
authority := strings.Join(strings.Split(t.Name(), "/"), "")
|
||||
ldsTargets := []string{
|
||||
"lds.target.good:0000",
|
||||
fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/lds.targer.good:1111", authority),
|
||||
}
|
||||
rdsTargets := []string{
|
||||
"route-config-0",
|
||||
fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/route-config-1", authority),
|
||||
}
|
||||
listeners := make([]*v3listenerpb.Listener, len(ldsTargets))
|
||||
listenerAnys := make([]*anypb.Any, len(ldsTargets))
|
||||
for i := range ldsTargets {
|
||||
listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i])
|
||||
listenerAnys[i] = testutils.MarshalAny(t, listeners[i])
|
||||
}
|
||||
|
||||
// Start two management servers.
|
||||
mgmtServer1 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
|
||||
mgmtServer2 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
|
||||
|
||||
// The first of the above management servers will be the top-level xDS
|
||||
// server in the bootstrap configuration, and the second will be the xDS
|
||||
// server corresponding to the test authority.
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
resourceTypes := map[string]xdsclient.ResourceType{}
|
||||
listenerType := listenerType
|
||||
resourceTypes[xdsresource.V3ListenerURL] = listenerType
|
||||
si1 := clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer1.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
si2 := clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer2.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{ServerIdentifier: si1}},
|
||||
Node: clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
ResourceTypes: resourceTypes,
|
||||
// Xdstp style resource names used in this test use a slash removed
|
||||
// version of t.Name as their authority, and the empty config
|
||||
// results in the top-level xds server configuration being used for
|
||||
// this authority.
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
authority: {XDSServers: []xdsclient.ServerConfig{{ServerIdentifier: si2}}},
|
||||
},
|
||||
}
|
||||
|
||||
// Create two xDS clients with the above config.
|
||||
client1, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client1.Close()
|
||||
client2, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client2.Close()
|
||||
|
||||
// Check the resource dump before configuring resources on the management server.
|
||||
// Dump resources and expect empty configs.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
wantNode := &v3corepb.Node{
|
||||
Id: nodeID,
|
||||
UserAgentName: "user-agent",
|
||||
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
|
||||
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
|
||||
}
|
||||
wantResp := &v3statuspb.ClientStatusResponse{
|
||||
Config: []*v3statuspb.ClientConfig{
|
||||
{
|
||||
Node: wantNode,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Register watches, the first xDS client watches old style resource names,
|
||||
// while the second xDS client watches new style resource names.
|
||||
client1.WatchResource(xdsresource.V3ListenerURL, ldsTargets[0], noopListenerWatcher{})
|
||||
client2.WatchResource(xdsresource.V3ListenerURL, ldsTargets[1], noopListenerWatcher{})
|
||||
|
||||
// Check the resource dump. Both clients should have all resources in
|
||||
// REQUESTED state.
|
||||
wantConfigs1 := []*v3statuspb.ClientConfig_GenericXdsConfig{
|
||||
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
|
||||
}
|
||||
wantConfigs2 := []*v3statuspb.ClientConfig_GenericXdsConfig{
|
||||
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
|
||||
}
|
||||
wantResp = &v3statuspb.ClientStatusResponse{
|
||||
Config: []*v3statuspb.ClientConfig{
|
||||
{
|
||||
Node: wantNode,
|
||||
GenericXdsConfigs: wantConfigs1,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantResp = &v3statuspb.ClientStatusResponse{
|
||||
Config: []*v3statuspb.ClientConfig{
|
||||
{
|
||||
Node: wantNode,
|
||||
GenericXdsConfigs: wantConfigs2,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Configure resources on the first management server.
|
||||
if err := mgmtServer1.Update(ctx, e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: listeners[:1],
|
||||
SkipValidation: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Check the resource dump. One client should have resources in ACKED state,
|
||||
// while the other should still have resources in REQUESTED state.
|
||||
wantConfigs1 = []*v3statuspb.ClientConfig_GenericXdsConfig{
|
||||
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[0], nil),
|
||||
}
|
||||
wantResp = &v3statuspb.ClientStatusResponse{
|
||||
Config: []*v3statuspb.ClientConfig{
|
||||
{
|
||||
Node: wantNode,
|
||||
GenericXdsConfigs: wantConfigs1,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantResp = &v3statuspb.ClientStatusResponse{
|
||||
Config: []*v3statuspb.ClientConfig{
|
||||
{
|
||||
Node: wantNode,
|
||||
GenericXdsConfigs: wantConfigs2,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Configure resources on the second management server.
|
||||
if err := mgmtServer2.Update(ctx, e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: listeners[1:],
|
||||
SkipValidation: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Check the resource dump. Both clients should have appropriate resources
|
||||
// in REQUESTED state.
|
||||
wantConfigs2 = []*v3statuspb.ClientConfig_GenericXdsConfig{
|
||||
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil),
|
||||
}
|
||||
wantResp = &v3statuspb.ClientStatusResponse{
|
||||
Config: []*v3statuspb.ClientConfig{
|
||||
{
|
||||
Node: wantNode,
|
||||
GenericXdsConfigs: wantConfigs1,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantResp = &v3statuspb.ClientStatusResponse{
|
||||
Config: []*v3statuspb.ClientConfig{
|
||||
{
|
||||
Node: wantNode,
|
||||
GenericXdsConfigs: wantConfigs2,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests that when a resource update is NACKed by the xDS client, the resource
|
||||
// dump mentions that the xDS client is using the old good version of the
|
||||
// resource, but contains the error state for the most recent bad update.
|
||||
func (s) TestDumpResources_ACK_NACK(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Spin up an xDS management server on a local port.
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
|
||||
|
||||
nodeID := uuid.New().String()
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
|
||||
|
||||
// Start watching a listener resource.
|
||||
client.WatchResource(xdsresource.V3ListenerURL, ldsName, noopListenerWatcher{})
|
||||
|
||||
// Configure the management server with a good version of the listener resource.
|
||||
goodListener := e2e.DefaultClientListener(ldsName, rdsName)
|
||||
if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{goodListener},
|
||||
SkipValidation: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Dump resources and expect ACK config with version "1".
|
||||
wantNode := &v3corepb.Node{
|
||||
Id: nodeID,
|
||||
UserAgentName: "user-agent",
|
||||
UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
|
||||
ClientFeatures: []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
|
||||
}
|
||||
goodListenerAny := testutils.MarshalAny(t, goodListener)
|
||||
wantConfigs := []*v3statuspb.ClientConfig_GenericXdsConfig{
|
||||
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsName, "1", v3adminpb.ClientResourceStatus_ACKED, goodListenerAny, nil),
|
||||
}
|
||||
wantResp := &v3statuspb.ClientStatusResponse{
|
||||
Config: []*v3statuspb.ClientConfig{
|
||||
{
|
||||
Node: wantNode,
|
||||
GenericXdsConfigs: wantConfigs,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Update the management server with a bad version of the listener resource
|
||||
// which is expected to be NACKed by the xDS client.
|
||||
badListener := badListenerResource(t, ldsName)
|
||||
if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{badListener},
|
||||
SkipValidation: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Dump resources and verify that the xDS client is using the old good
|
||||
// version of the resource ("1"), but contains the error state for the most
|
||||
// recent bad update ("2").
|
||||
wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{
|
||||
makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsName, "1", v3adminpb.ClientResourceStatus_NACKED, goodListenerAny, &v3adminpb.UpdateFailureState{VersionInfo: "2"}),
|
||||
}
|
||||
wantResp = &v3statuspb.ClientStatusResponse{
|
||||
Config: []*v3statuspb.ClientConfig{
|
||||
{
|
||||
Node: wantNode,
|
||||
GenericXdsConfigs: wantConfigs,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := checkResourceDump(ctx, wantResp, client); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
390
internal/xds/clients/xdsclient/test/helpers_test.go
Normal file
390
internal/xds/clients/xdsclient/test/helpers_test.go
Normal file
@@ -0,0 +1,390 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc/internal/grpctest"
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/buffer"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/pretty"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
|
||||
v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
)
|
||||
|
||||
type s struct {
|
||||
grpctest.Tester
|
||||
}
|
||||
|
||||
func Test(t *testing.T) {
|
||||
grpctest.RunSubTests(t, s{})
|
||||
}
|
||||
|
||||
const (
|
||||
defaultTestWatchExpiryTimeout = 500 * time.Millisecond
|
||||
defaultTestTimeout = 10 * time.Second
|
||||
defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen.
|
||||
|
||||
// ListenerResourceTypeName represents the transport agnostic name for the
|
||||
// listener resource.
|
||||
listenerResourceTypeName = "ListenerResource"
|
||||
|
||||
ldsName = "xdsclient-test-lds-resource"
|
||||
rdsName = "xdsclient-test-rds-resource"
|
||||
ldsNameNewStyle = "xdstp:///envoy.config.listener.v3.Listener/xdsclient-test-lds-resource"
|
||||
rdsNameNewStyle = "xdstp:///envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource"
|
||||
)
|
||||
|
||||
var (
|
||||
// Singleton instantiation of the resource type implementation.
|
||||
listenerType = xdsclient.ResourceType{
|
||||
TypeURL: xdsresource.V3ListenerURL,
|
||||
TypeName: listenerResourceTypeName,
|
||||
AllResourcesRequiredInSotW: true,
|
||||
Decoder: listenerDecoder{},
|
||||
}
|
||||
)
|
||||
|
||||
func unmarshalListenerResource(rProto *anypb.Any) (string, listenerUpdate, error) {
|
||||
rProto, err := xdsresource.UnwrapResource(rProto)
|
||||
if err != nil {
|
||||
return "", listenerUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err)
|
||||
}
|
||||
if !xdsresource.IsListenerResource(rProto.GetTypeUrl()) {
|
||||
return "", listenerUpdate{}, fmt.Errorf("unexpected listener resource type: %q", rProto.GetTypeUrl())
|
||||
}
|
||||
lis := &v3listenerpb.Listener{}
|
||||
if err := proto.Unmarshal(rProto.GetValue(), lis); err != nil {
|
||||
return "", listenerUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
|
||||
}
|
||||
|
||||
lu, err := processListener(lis)
|
||||
if err != nil {
|
||||
return lis.GetName(), listenerUpdate{}, err
|
||||
}
|
||||
lu.Raw = rProto.GetValue()
|
||||
return lis.GetName(), *lu, nil
|
||||
}
|
||||
|
||||
func processListener(lis *v3listenerpb.Listener) (*listenerUpdate, error) {
|
||||
if lis.GetApiListener() != nil {
|
||||
return processClientSideListener(lis)
|
||||
}
|
||||
return processServerSideListener(lis)
|
||||
}
|
||||
|
||||
// processClientSideListener checks if the provided Listener proto meets
|
||||
// the expected criteria. If so, it returns a non-empty routeConfigName.
|
||||
func processClientSideListener(lis *v3listenerpb.Listener) (*listenerUpdate, error) {
|
||||
update := &listenerUpdate{}
|
||||
|
||||
apiLisAny := lis.GetApiListener().GetApiListener()
|
||||
if !xdsresource.IsHTTPConnManagerResource(apiLisAny.GetTypeUrl()) {
|
||||
return nil, fmt.Errorf("unexpected http connection manager resource type: %q", apiLisAny.GetTypeUrl())
|
||||
}
|
||||
apiLis := &v3httppb.HttpConnectionManager{}
|
||||
if err := proto.Unmarshal(apiLisAny.GetValue(), apiLis); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal api_listener: %v", err)
|
||||
}
|
||||
|
||||
switch apiLis.RouteSpecifier.(type) {
|
||||
case *v3httppb.HttpConnectionManager_Rds:
|
||||
if configsource := apiLis.GetRds().GetConfigSource(); configsource.GetAds() == nil && configsource.GetSelf() == nil {
|
||||
return nil, fmt.Errorf("LDS's RDS configSource is not ADS or Self: %+v", lis)
|
||||
}
|
||||
name := apiLis.GetRds().GetRouteConfigName()
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("empty route_config_name: %+v", lis)
|
||||
}
|
||||
update.RouteConfigName = name
|
||||
case *v3httppb.HttpConnectionManager_RouteConfig:
|
||||
routeU := apiLis.GetRouteConfig()
|
||||
if routeU == nil {
|
||||
return nil, fmt.Errorf("empty inline RDS resp:: %+v", lis)
|
||||
}
|
||||
if routeU.Name == "" {
|
||||
return nil, fmt.Errorf("empty route_config_name in inline RDS resp: %+v", lis)
|
||||
}
|
||||
update.RouteConfigName = routeU.Name
|
||||
case nil:
|
||||
return nil, fmt.Errorf("no RouteSpecifier: %+v", apiLis)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported type %T for RouteSpecifier", apiLis.RouteSpecifier)
|
||||
}
|
||||
|
||||
return update, nil
|
||||
}
|
||||
|
||||
func processServerSideListener(lis *v3listenerpb.Listener) (*listenerUpdate, error) {
|
||||
if n := len(lis.ListenerFilters); n != 0 {
|
||||
return nil, fmt.Errorf("unsupported field 'listener_filters' contains %d entries", n)
|
||||
}
|
||||
if lis.GetUseOriginalDst().GetValue() {
|
||||
return nil, errors.New("unsupported field 'use_original_dst' is present and set to true")
|
||||
}
|
||||
addr := lis.GetAddress()
|
||||
if addr == nil {
|
||||
return nil, fmt.Errorf("no address field in LDS response: %+v", lis)
|
||||
}
|
||||
sockAddr := addr.GetSocketAddress()
|
||||
if sockAddr == nil {
|
||||
return nil, fmt.Errorf("no socket_address field in LDS response: %+v", lis)
|
||||
}
|
||||
lu := &listenerUpdate{
|
||||
InboundListenerCfg: &inboundListenerConfig{
|
||||
Address: sockAddr.GetAddress(),
|
||||
Port: strconv.Itoa(int(sockAddr.GetPortValue())),
|
||||
},
|
||||
}
|
||||
|
||||
return lu, nil
|
||||
}
|
||||
|
||||
type listenerDecoder struct{}
|
||||
|
||||
// Decode deserializes and validates an xDS resource serialized inside the
|
||||
// provided `Any` proto, as received from the xDS management server.
|
||||
func (listenerDecoder) Decode(resource *xdsclient.AnyProto, _ xdsclient.DecodeOptions) (*xdsclient.DecodeResult, error) {
|
||||
name, listener, err := unmarshalListenerResource(resource.ToAny())
|
||||
switch {
|
||||
case name == "":
|
||||
// Name is unset only when protobuf deserialization fails.
|
||||
return nil, err
|
||||
case err != nil:
|
||||
// Protobuf deserialization succeeded, but resource validation failed.
|
||||
return &xdsclient.DecodeResult{Name: name, Resource: &listenerResourceData{Resource: listenerUpdate{}}}, err
|
||||
}
|
||||
|
||||
return &xdsclient.DecodeResult{Name: name, Resource: &listenerResourceData{Resource: listener}}, nil
|
||||
|
||||
}
|
||||
|
||||
// listenerResourceData wraps the configuration of a Listener resource as
|
||||
// received from the management server.
|
||||
//
|
||||
// Implements the ResourceData interface.
|
||||
type listenerResourceData struct {
|
||||
xdsclient.ResourceData
|
||||
|
||||
Resource listenerUpdate
|
||||
}
|
||||
|
||||
// Equal returns true if other is equal to l.
|
||||
func (l *listenerResourceData) Equal(other xdsclient.ResourceData) bool {
|
||||
if l == nil && other == nil {
|
||||
return true
|
||||
}
|
||||
if (l == nil) != (other == nil) {
|
||||
return false
|
||||
}
|
||||
return bytes.Equal(l.Resource.Raw, other.Bytes())
|
||||
}
|
||||
|
||||
// ToJSON returns a JSON string representation of the resource data.
|
||||
func (l *listenerResourceData) ToJSON() string {
|
||||
return pretty.ToJSON(l.Resource)
|
||||
}
|
||||
|
||||
// Bytes returns the underlying raw protobuf form of the listener resource.
|
||||
func (l *listenerResourceData) Bytes() []byte {
|
||||
return l.Resource.Raw
|
||||
}
|
||||
|
||||
// ListenerUpdate contains information received in an LDS response, which is of
|
||||
// interest to the registered LDS watcher.
|
||||
type listenerUpdate struct {
|
||||
// RouteConfigName is the route configuration name corresponding to the
|
||||
// target which is being watched through LDS.
|
||||
RouteConfigName string
|
||||
|
||||
// InboundListenerCfg contains inbound listener configuration.
|
||||
InboundListenerCfg *inboundListenerConfig
|
||||
|
||||
// Raw is the resource from the xds response.
|
||||
Raw []byte
|
||||
}
|
||||
|
||||
// InboundListenerConfig contains information about the inbound listener, i.e
|
||||
// the server-side listener.
|
||||
type inboundListenerConfig struct {
|
||||
// Address is the local address on which the inbound listener is expected to
|
||||
// accept incoming connections.
|
||||
Address string
|
||||
// Port is the local port on which the inbound listener is expected to
|
||||
// accept incoming connections.
|
||||
Port string
|
||||
}
|
||||
|
||||
func makeAuthorityName(name string) string {
|
||||
segs := strings.Split(name, "/")
|
||||
return strings.Join(segs, "")
|
||||
}
|
||||
|
||||
func makeNewStyleLDSName(authority string) string {
|
||||
return fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/xdsclient-test-lds-resource", authority)
|
||||
}
|
||||
|
||||
// buildResourceName returns the resource name in the format of an xdstp://
|
||||
// resource.
|
||||
func buildResourceName(typeName, auth, id string, ctxParams map[string]string) string {
|
||||
var typS string
|
||||
switch typeName {
|
||||
case listenerResourceTypeName:
|
||||
typS = "envoy.config.listener.v3.Listener"
|
||||
default:
|
||||
// If the name doesn't match any of the standard resources fallback
|
||||
// to the type name.
|
||||
typS = typeName
|
||||
}
|
||||
return (&xdsresource.Name{
|
||||
Scheme: "xdstp",
|
||||
Authority: auth,
|
||||
Type: typS,
|
||||
ID: id,
|
||||
ContextParams: ctxParams,
|
||||
}).String()
|
||||
}
|
||||
|
||||
// testMetricsReporter is a MetricsReporter to be used in tests. It sends
|
||||
// recording events on channels and provides helpers to check if certain events
|
||||
// have taken place.
|
||||
type testMetricsReporter struct {
|
||||
metricsCh *buffer.Unbounded[any]
|
||||
|
||||
mu sync.Mutex
|
||||
asyncReporters map[clients.AsyncReporter]struct{}
|
||||
}
|
||||
|
||||
// newTestMetricsReporter returns a new testMetricsReporter.
|
||||
func newTestMetricsReporter() *testMetricsReporter {
|
||||
return &testMetricsReporter{
|
||||
metricsCh: buffer.NewUnbounded[any](),
|
||||
asyncReporters: make(map[clients.AsyncReporter]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// waitForMetric waits for a metric to be recorded and verifies that the
|
||||
// recorded metrics data matches the expected metricsDataWant. Returns
|
||||
// an error if failed to wait or received wrong data.
|
||||
func (r *testMetricsReporter) waitForMetric(ctx context.Context, metricsDataWant any) error {
|
||||
select {
|
||||
case got := <-r.metricsCh.Get():
|
||||
r.metricsCh.Load()
|
||||
if diff := cmp.Diff(got, metricsDataWant); diff != "" {
|
||||
return fmt.Errorf("received unexpected metrics value (-got, +want): %v", diff)
|
||||
}
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("timeout waiting for metric check: %v", ctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
// waitForSpecificMetric waits for a specific metric to be recorded, ignoring
|
||||
// and discarding any other metrics received in the meantime. Returns an error
|
||||
// if the context expires before the requested metric is received.
|
||||
//
|
||||
// This is necessary when multiple distinct metrics (e.g., ServerFailure,
|
||||
// ResourceUpdateValid, and XDSClientConnected) might be emitted asynchronously
|
||||
// and concurrently. Using waitForMetric would fail the test if an unrelated
|
||||
// metric happens to be at the front of the channel.
|
||||
func (r *testMetricsReporter) waitForSpecificMetric(ctx context.Context, metricsDataWant any) error {
|
||||
for {
|
||||
select {
|
||||
case got := <-r.metricsCh.Get():
|
||||
r.metricsCh.Load()
|
||||
if diff := cmp.Diff(got, metricsDataWant); diff == "" {
|
||||
return nil
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("timeout waiting for specific metric: %v", ctx.Err())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Receive waits for a metric to be recorded and returns it. Returns an error
|
||||
// if the context expires before a metric is received.
|
||||
func (r *testMetricsReporter) Receive(ctx context.Context) (any, error) {
|
||||
select {
|
||||
case got := <-r.metricsCh.Get():
|
||||
r.metricsCh.Load()
|
||||
return got, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// Drain clears all accumulated metrics from the channel.
|
||||
func (r *testMetricsReporter) Drain() {
|
||||
for {
|
||||
select {
|
||||
case <-r.metricsCh.Get():
|
||||
r.metricsCh.Load()
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ReportMetric sends the metrics data to the metricsCh channel.
|
||||
func (r *testMetricsReporter) ReportMetric(m any) {
|
||||
r.metricsCh.Put(m)
|
||||
}
|
||||
|
||||
func (r *testMetricsReporter) RegisterAsyncReporter(reporter clients.AsyncReporter) func() {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.asyncReporters[reporter] = struct{}{}
|
||||
return sync.OnceFunc(func() {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
delete(r.asyncReporters, reporter)
|
||||
})
|
||||
}
|
||||
|
||||
// triggerAsyncMetrics invokes the Report method on all registered asynchronous
|
||||
// metric reporters. This allows tests to explicitly compel the emission of
|
||||
// async gauge metrics at specific precise points in time for verification.
|
||||
func (r *testMetricsReporter) triggerAsyncMetrics() {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for reporter := range r.asyncReporters {
|
||||
reporter.Report(r)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *testMetricsReporter) numAsyncReporters() int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return len(r.asyncReporters)
|
||||
}
|
||||
1402
internal/xds/clients/xdsclient/test/lds_watchers_test.go
Normal file
1402
internal/xds/clients/xdsclient/test/lds_watchers_test.go
Normal file
File diff suppressed because it is too large
Load Diff
911
internal/xds/clients/xdsclient/test/metrics_test.go
Normal file
911
internal/xds/clients/xdsclient/test/metrics_test.go
Normal file
@@ -0,0 +1,911 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/internal/testutils"
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
"google.golang.org/grpc/internal/xds/clients/grpctransport"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/metrics"
|
||||
|
||||
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
|
||||
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
|
||||
)
|
||||
|
||||
// TestResourceUpdateMetrics configures an xDS client, and a management server
|
||||
// to send valid and invalid LDS updates, and verifies that the expected metrics
|
||||
// for both good and bad updates are emitted.
|
||||
func (s) TestResourceUpdateMetrics(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
tmr := newTestMetricsReporter()
|
||||
l, err := net.Listen("tcp", "localhost:0")
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen() failed: %v", err)
|
||||
}
|
||||
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: l})
|
||||
const listenerResourceName = "test-listener-resource"
|
||||
const routeConfigurationName = "test-route-configuration-resource"
|
||||
nodeID := uuid.New().String()
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
|
||||
resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
|
||||
si := clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
ResourceTypes: resourceTypes,
|
||||
// Xdstp resource names used in this test do not specify an
|
||||
// authority. These will end up looking up an entry with the
|
||||
// empty key in the authorities map. Having an entry with an
|
||||
// empty key and empty configuration, results in these
|
||||
// resources also using the top-level configuration.
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
"": {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
MetricsReporter: tmr,
|
||||
}
|
||||
// Create an xDS client with the above config.
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// Watch the valid listener configured on the management server. This should
|
||||
// cause a resource update valid metric to emit eventually.
|
||||
client.WatchResource(listenerType.TypeURL, listenerResourceName, noopListenerWatcher{})
|
||||
if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Update management server with a bad update. This should cause a resource
|
||||
// update invalid metric to emit eventually.
|
||||
resources = e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)},
|
||||
SkipValidation: true,
|
||||
}
|
||||
resources.Listeners[0].ApiListener = nil
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateInvalid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Resource update valid metric should have not emitted.
|
||||
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
||||
defer sCancel()
|
||||
if err := tmr.waitForMetric(sCtx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err == nil {
|
||||
t.Fatal("tmr.WaitForInt64Count(ctx, mdWant) succeeded when expected to timeout.")
|
||||
}
|
||||
}
|
||||
|
||||
// TestServerFailureMetrics_BeforeResponseRecv configures an xDS client, and a
|
||||
// management server. It then register a watcher and stops the management
|
||||
// server before sending a resource update, and verifies that the expected
|
||||
// metric for server failure is emitted.
|
||||
func (s) TestServerFailureMetrics_BeforeResponseRecv(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
tmr := newTestMetricsReporter()
|
||||
l, err := net.Listen("tcp", "localhost:0")
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen() failed: %v", err)
|
||||
}
|
||||
|
||||
lis := testutils.NewRestartableListener(l)
|
||||
streamOpened := make(chan struct{}, 1)
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
|
||||
Listener: lis,
|
||||
OnStreamOpen: func(context.Context, int64, string) error {
|
||||
select {
|
||||
case streamOpened <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
|
||||
si := clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
ResourceTypes: resourceTypes,
|
||||
// Xdstp resource names used in this test do not specify an
|
||||
// authority. These will end up looking up an entry with the
|
||||
// empty key in the authorities map. Having an entry with an
|
||||
// empty key and empty configuration, results in these
|
||||
// resources also using the top-level configuration.
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
"": {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
MetricsReporter: tmr,
|
||||
}
|
||||
// Create an xDS client with the above config.
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
const listenerResourceName = "test-listener-resource"
|
||||
|
||||
// Watch for the listener on the above management server.
|
||||
client.WatchResource(listenerType.TypeURL, listenerResourceName, noopListenerWatcher{})
|
||||
// Verify that an ADS stream is opened and an LDS request with the above
|
||||
// resource name is sent.
|
||||
select {
|
||||
case <-streamOpened:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Timeout when waiting for ADS stream to open")
|
||||
}
|
||||
|
||||
// Close the listener and ensure that the ADS stream breaks. This should
|
||||
// cause a server failure metric to emit eventually.
|
||||
lis.Stop()
|
||||
|
||||
// Restart to prevent the attempt to create a new ADS stream after back off.
|
||||
lis.Restart()
|
||||
|
||||
if err := tmr.waitForMetric(ctx, &metrics.ServerFailure{ServerURI: mgmtServer.Address}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestServerFailureMetrics_AfterResponseRecv configures an xDS client and a
|
||||
// management server to send a valid LDS update, and verifies that the
|
||||
// successful update metric is emitted. When the client ACKs the update, the
|
||||
// server returns an error, breaking the stream. The test then verifies that the
|
||||
// server failure metric is not emitted, because the ADS stream was closed after
|
||||
// a response was received on the stream. Finally, the test waits for the client
|
||||
// to establish a new stream and verifies that the client emits a metric after
|
||||
// receiving a successful update.
|
||||
func (s) TestServerFailureMetrics_AfterResponseRecv(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
tmr := newTestMetricsReporter()
|
||||
l, err := testutils.LocalTCPListener()
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen() failed: %v", err)
|
||||
}
|
||||
lis := testutils.NewRestartableListener(l)
|
||||
streamCreationQuota := make(chan struct{}, 1)
|
||||
streamCreationQuota <- struct{}{}
|
||||
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
|
||||
Listener: lis,
|
||||
OnStreamOpen: func(context.Context, int64, string) error {
|
||||
// The following select block is used to block stream creation after
|
||||
// the first stream has failed, but while we are waiting to verify
|
||||
// that the failure metric is not reported.
|
||||
select {
|
||||
case <-streamCreationQuota:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
return nil
|
||||
},
|
||||
OnStreamRequest: func(streamID int64, req *v3discoverypb.DiscoveryRequest) error {
|
||||
// We only want the ACK on the first stream to return an error
|
||||
// (leading to stream closure), without effecting subsequent stream
|
||||
// attempts.
|
||||
if streamID == 1 && req.GetVersionInfo() != "" {
|
||||
return errors.New("test configured error")
|
||||
}
|
||||
return nil
|
||||
}},
|
||||
)
|
||||
const listenerResourceName = "test-listener-resource"
|
||||
const routeConfigurationName = "test-route-configuration-resource"
|
||||
nodeID := uuid.New().String()
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
|
||||
resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
|
||||
si := clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
ResourceTypes: resourceTypes,
|
||||
// Xdstp resource names used in this test do not specify an
|
||||
// authority. These will end up looking up an entry with the
|
||||
// empty key in the authorities map. Having an entry with an
|
||||
// empty key and empty configuration, results in these
|
||||
// resources also using the top-level configuration.
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
"": {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
MetricsReporter: tmr,
|
||||
}
|
||||
// Create an xDS client with the above config.
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// Watch the valid listener configured on the management server. This should
|
||||
// cause a resource update valid metric to emit eventually.
|
||||
client.WatchResource(listenerType.TypeURL, listenerResourceName, noopListenerWatcher{})
|
||||
if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// When the client sends an ACK, the management server would reply with an
|
||||
// error, breaking the stream.
|
||||
// Server failure should still have no recording point.
|
||||
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
||||
defer sCancel()
|
||||
failureMetric := &metrics.ServerFailure{ServerURI: mgmtServer.Address}
|
||||
if err := tmr.waitForMetric(sCtx, failureMetric); err == nil {
|
||||
t.Fatalf("tmr.waitForMetric(%v) succeeded when expected to timeout.", failureMetric)
|
||||
} else if sCtx.Err() == nil {
|
||||
t.Fatalf("tmr.WaitForInt64Count(%v) = %v, want context deadline exceeded", failureMetric, err)
|
||||
}
|
||||
// Unblock stream creation and verify that an update is received
|
||||
// successfully.
|
||||
close(streamCreationQuota)
|
||||
if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConnectedMetric verifies the "grpc.xds_client.connected" metric state
|
||||
// transitions. It begins by ensuring no metrics are reported before connection
|
||||
// is attempted. Then it establishes a connection by watching a valid resource
|
||||
// and verifies the connected state pulses to 1. Finally, it stops the
|
||||
// management server and verifies the state drops back to 0.
|
||||
func (s) TestConnectedMetric(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
tmr := newTestMetricsReporter()
|
||||
l, err := net.Listen("tcp", "localhost:0")
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen() failed: %v", err)
|
||||
}
|
||||
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: l})
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{
|
||||
ServerIdentifier: clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
},
|
||||
}},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{
|
||||
"insecure": {Credentials: insecure.NewBundle()},
|
||||
}),
|
||||
ResourceTypes: map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType},
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
"": {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
MetricsReporter: tmr,
|
||||
}
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
tmr.triggerAsyncMetrics()
|
||||
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
||||
defer sCancel()
|
||||
if err := tmr.waitForSpecificMetric(sCtx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address}); err == nil {
|
||||
t.Fatal("XDSClientConnected metric reported before any watch was started")
|
||||
}
|
||||
|
||||
const listenerName = "test-listener-resource"
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, "route-config")},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server: %v", err)
|
||||
}
|
||||
client.WatchResource(listenerType.TypeURL, listenerName, noopListenerWatcher{})
|
||||
|
||||
// Wait for the update to ensure we are connected.
|
||||
if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Now trigger async metrics.
|
||||
tmr.triggerAsyncMetrics()
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mgmtServer.Stop()
|
||||
|
||||
// Wait for the synchronous server failure metric to confirm disconnect.
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.ServerFailure{ServerURI: mgmtServer.Address}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tmr.triggerAsyncMetrics()
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 0}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify async metric reporters are unregistered when client closes.
|
||||
client.Close()
|
||||
|
||||
if count := tmr.numAsyncReporters(); count != 0 {
|
||||
t.Fatalf("Async reporter not unregistered after client close, count: %d", count)
|
||||
}
|
||||
|
||||
// Drain the channel of any leftover metrics from previous pulses.
|
||||
tmr.Drain()
|
||||
|
||||
tmr.triggerAsyncMetrics()
|
||||
// No metrics should be reported now because there are no reporters.
|
||||
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
|
||||
defer sCancel()
|
||||
if _, err := tmr.Receive(sCtx); err == nil {
|
||||
t.Fatal("Metrics reported after all reporters were unregistered")
|
||||
}
|
||||
}
|
||||
|
||||
// TestResourceMetrics verifies that the xDS client correctly tracks resource
|
||||
// states (acked, nacked_but_cached) in "grpc.xds_client.resources" metric. It
|
||||
// watches a resource, pushes a valid update from the management server, and
|
||||
// asserts that the resource transitions to 'acked' state. Then it pushes an
|
||||
// invalid update and asserts that the resource transitions to
|
||||
// 'nacked_but_cached' state.
|
||||
func (s) TestResourceMetrics(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
tmr := newTestMetricsReporter()
|
||||
l, err := net.Listen("tcp", "localhost:0")
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen() failed: %v", err)
|
||||
}
|
||||
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: l})
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{
|
||||
ServerIdentifier: clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
},
|
||||
}},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{
|
||||
"insecure": {Credentials: insecure.NewBundle()},
|
||||
}),
|
||||
ResourceTypes: map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType},
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
"": {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
MetricsReporter: tmr,
|
||||
}
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
const listenerName = "test-listener-resource"
|
||||
const routeConfigName = "test-route-configuration-resource"
|
||||
|
||||
client.WatchResource(listenerType.TypeURL, listenerName, noopListenerWatcher{})
|
||||
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeConfigName)},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server: %v", err)
|
||||
}
|
||||
|
||||
if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Trigger async metrics.
|
||||
tmr.triggerAsyncMetrics()
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{
|
||||
Authority: "#old", // Default value for old-style non-xdstp as per gRFC A78
|
||||
ResourceType: "ListenerResource",
|
||||
CacheState: "acked",
|
||||
Count: 1,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
resources.Listeners[0].ApiListener = nil
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server: %v", err)
|
||||
}
|
||||
|
||||
// Wait for Invalid update.
|
||||
if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateInvalid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tmr.triggerAsyncMetrics()
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{
|
||||
Authority: "#old",
|
||||
ResourceType: "ListenerResource",
|
||||
CacheState: "nacked_but_cached",
|
||||
Count: 1,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResourceMetrics_Extended verifies complex resource state transitions
|
||||
// (requested, does_not_exist, nacked) in "grpc.xds_client.resources" metric
|
||||
// across multiple resources. It watches several resources, pushes a partial
|
||||
// update (some valid, some invalid) to transition them to 'acked' and 'nacked'
|
||||
// states, then management server removes a resource and asserts that active
|
||||
// watchers transition back to 'requested' while omitted resources transition
|
||||
// to 'does_not_exist'.
|
||||
func (s) TestResourceMetrics_Extended(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
tmr := newTestMetricsReporter()
|
||||
l, err := net.Listen("tcp", "localhost:0")
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen() failed: %v", err)
|
||||
}
|
||||
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: l})
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{
|
||||
ServerIdentifier: clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
},
|
||||
}},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{
|
||||
"insecure": {Credentials: insecure.NewBundle()},
|
||||
}),
|
||||
ResourceTypes: map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType},
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
"": {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
MetricsReporter: tmr,
|
||||
}
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
resRequested1 := "res-requested-1"
|
||||
resRequested2 := "res-requested-2"
|
||||
resNacked1 := "res-nacked-1"
|
||||
resNacked2 := "res-nacked-2"
|
||||
resRemoved := "res-not-exist"
|
||||
|
||||
resList := []string{resRequested1, resRequested2, resNacked1, resNacked2, resRemoved}
|
||||
for _, res := range resList {
|
||||
client.WatchResource(listenerType.TypeURL, res, noopListenerWatcher{})
|
||||
}
|
||||
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{
|
||||
e2e.DefaultClientListener(resNacked1, "route-config"),
|
||||
e2e.DefaultClientListener(resNacked2, "route-config"),
|
||||
e2e.DefaultClientListener(resRemoved, "route-config"),
|
||||
},
|
||||
SkipValidation: true,
|
||||
}
|
||||
resources.Listeners[0].ApiListener = nil
|
||||
resources.Listeners[1].ApiListener = nil
|
||||
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server: %v", err)
|
||||
}
|
||||
|
||||
// Wait for the resource update to be accepted.
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Push an empty update to remove all resources from the management server.
|
||||
resourcesEmpty := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resourcesEmpty); err != nil {
|
||||
t.Fatalf("Failed to update management server: %v", err)
|
||||
}
|
||||
|
||||
tmr.triggerAsyncMetrics()
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{
|
||||
Authority: "#old",
|
||||
ResourceType: "ListenerResource",
|
||||
CacheState: "requested",
|
||||
Count: 2,
|
||||
}); err != nil {
|
||||
t.Fatalf("Failed to verify requested count: %v", err)
|
||||
}
|
||||
|
||||
tmr.triggerAsyncMetrics()
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{
|
||||
Authority: "#old",
|
||||
ResourceType: "ListenerResource",
|
||||
CacheState: "nacked",
|
||||
Count: 2,
|
||||
}); err != nil {
|
||||
t.Fatalf("Failed to verify nacked count: %v", err)
|
||||
}
|
||||
|
||||
// Wait for the xDS client to process the empty update and trigger
|
||||
// ResourceNotFound for omitted resources immediately.
|
||||
lw := newListenerWatcher()
|
||||
client.WatchResource(listenerType.TypeURL, resRemoved, lw)
|
||||
|
||||
// Verify that resources missing from the authoritative update transition to
|
||||
// the does_not_exist state.
|
||||
if err := verifyResourceErrorType(ctx, lw.resourceErrCh, xdsresource.ErrorTypeResourceNotFound, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tmr.triggerAsyncMetrics()
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{
|
||||
Authority: "#old",
|
||||
ResourceType: "ListenerResource",
|
||||
CacheState: "does_not_exist",
|
||||
Count: 1,
|
||||
}); err != nil {
|
||||
t.Fatalf("Failed to verify does_not_exist count: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConnectedMetric_Reconnection verifies "grpc.xds_client.connected" metric
|
||||
// accuracy during flaky network conditions. It uses a restartable listener and
|
||||
// a dial interceptor to pause connection attempts. It asserts that the
|
||||
// connected state pulses to 1 when a stream succeeds, remains 1 immediately
|
||||
// after failures before retries, transitions to 0 after subsequent failures,
|
||||
// and returns to 1 only after successful stream recreation.
|
||||
func (s) TestConnectedMetric_Reconnection(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
l, err := testutils.LocalTCPListener()
|
||||
if err != nil {
|
||||
t.Fatalf("net.Listen() failed: %v", err)
|
||||
}
|
||||
lis := testutils.NewRestartableListener(l)
|
||||
|
||||
sendResponse := make(chan struct{})
|
||||
streamOpened := make(chan struct{}, 10)
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
|
||||
Listener: lis,
|
||||
OnStreamOpen: func(ctx context.Context, _ int64, _ string) error {
|
||||
select {
|
||||
case streamOpened <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
return nil
|
||||
},
|
||||
OnStreamResponse: func(ctx context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, _ *v3discoverypb.DiscoveryResponse) {
|
||||
select {
|
||||
case <-sendResponse:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
},
|
||||
})
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
|
||||
si := clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
// streamAttemptCh is used by the interceptor to notify the test that a
|
||||
// stream is attempting to open.
|
||||
streamAttemptCh := testutils.NewChannel()
|
||||
// commandCh is used by the test to tell the interceptor whether to block or pass.
|
||||
commandCh := testutils.NewChannel()
|
||||
// blocked is used by the interceptor to signal the test that it has
|
||||
// successfully blocked the stream creation attempt.
|
||||
blocked := make(chan struct{})
|
||||
// unblock is used by the test to release the interceptor and allow the
|
||||
// stream creation to proceed.
|
||||
unblock := make(chan struct{})
|
||||
|
||||
// customGRPCNewClient overrides the gRPC client creation to inject a stream
|
||||
// interceptor that can block ADS stream creation attempts for testing
|
||||
// reconnection behavior.
|
||||
customGRPCNewClient := func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
|
||||
interceptor := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, cops ...grpc.CallOption) (grpc.ClientStream, error) {
|
||||
streamAttemptCh.Send(struct{}{}) // Notify test
|
||||
|
||||
cmd, err := commandCh.Receive(ctx) // Wait for command
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if cmd.(string) == "block" {
|
||||
select {
|
||||
case blocked <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
// Pause stream creation until released by the test.
|
||||
<-unblock
|
||||
}
|
||||
return streamer(ctx, desc, cc, method, cops...)
|
||||
}
|
||||
opts = append(opts, grpc.WithStreamInterceptor(interceptor))
|
||||
return grpc.NewClient(target, opts...)
|
||||
}
|
||||
|
||||
tmr := newTestMetricsReporter()
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle(), GRPCNewClient: customGRPCNewClient}}
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
ResourceTypes: resourceTypes,
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
"": {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
MetricsReporter: tmr,
|
||||
}
|
||||
|
||||
waitForStreamSuccess := func() {
|
||||
for {
|
||||
select {
|
||||
case <-streamOpened:
|
||||
return
|
||||
case <-streamAttemptCh.C:
|
||||
commandCh.Send("pass")
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("Timeout waiting for stream success")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop listener to force initial connection failure.
|
||||
lis.Stop()
|
||||
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
const listenerName = "test-listener-resource"
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, "route-config")},
|
||||
SkipValidation: true,
|
||||
}
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server: %v", err)
|
||||
}
|
||||
|
||||
// Initiate watch to trigger connection attempt.
|
||||
client.WatchResource(listenerType.TypeURL, listenerName, noopListenerWatcher{})
|
||||
|
||||
tmr.triggerAsyncMetrics()
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 0}); err != nil {
|
||||
t.Fatalf("XDSClientConnected check failed at start - got: %v, want 0", err)
|
||||
}
|
||||
|
||||
// Verify state transitions to connected (Value: 1) when stream succeeds.
|
||||
lis.Restart()
|
||||
|
||||
waitForStreamSuccess()
|
||||
|
||||
tmr.triggerAsyncMetrics()
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 1}); err != nil {
|
||||
t.Fatalf("XDSClientConnected check failed after 1st NewStream - got: %v, want 1", err)
|
||||
}
|
||||
|
||||
// Push resource update to confirm the connection is functional.
|
||||
close(sendResponse)
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify state remains connected (Value: 1) immediately after failure before
|
||||
// NewStream retry.
|
||||
lis.Stop()
|
||||
|
||||
if _, err := streamAttemptCh.Receive(ctx); err != nil {
|
||||
t.Fatalf("Timeout waiting for 2nd stream attempt (block): %v", err)
|
||||
}
|
||||
commandCh.Send("block")
|
||||
|
||||
<-blocked
|
||||
|
||||
tmr.triggerAsyncMetrics()
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 1}); err != nil {
|
||||
t.Fatalf("XDSClientConnected check failed while NewStream is blocked - got: %v, want 1", err)
|
||||
}
|
||||
|
||||
// Verify state transitions to disconnected (Value: 0) after NewStream attempt
|
||||
// fails.
|
||||
close(unblock)
|
||||
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.ServerFailure{ServerURI: mgmtServer.Address}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tmr.triggerAsyncMetrics()
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 0}); err != nil {
|
||||
t.Fatalf("XDSClientConnected check failed after NewStream failure - got: %v, want 0", err)
|
||||
}
|
||||
|
||||
sendResponse = make(chan struct{})
|
||||
|
||||
// Verify state remains disconnected (Value: 0) while waiting for the first
|
||||
// response on the new stream.
|
||||
lis.Restart()
|
||||
|
||||
// Clear channels to ensure fresh readings.
|
||||
for len(streamOpened) > 0 {
|
||||
<-streamOpened
|
||||
}
|
||||
|
||||
waitForStreamSuccess()
|
||||
|
||||
tmr.Drain()
|
||||
tmr.triggerAsyncMetrics()
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 0}); err != nil {
|
||||
t.Fatalf("XDSClientConnected check failed after successful NewStream but before response - got: %v, want 0", err)
|
||||
}
|
||||
|
||||
// Verify state returns to connected (Value: 1) after receiving the first
|
||||
// response on the new stream.
|
||||
close(sendResponse)
|
||||
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tmr.triggerAsyncMetrics()
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 1}); err != nil {
|
||||
t.Fatalf("XDSClientConnected check failed after response- got: %v, want 1", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResourceMetrics_AuthorityOldStyle verifies that the xDS client correctly
|
||||
// falls back to '#old' in "grpc.xds_client.resources" metrics for legacy
|
||||
// authorities. It watches a resource using default authority and asserts that
|
||||
// the metric is reported under '#old' instead of an empty string.
|
||||
func (s) TestResourceMetrics_AuthorityOldStyle(t *testing.T) {
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
serverCfg := xdsclient.ServerConfig{
|
||||
ServerIdentifier: clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
},
|
||||
}
|
||||
|
||||
tmr := newTestMetricsReporter()
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{serverCfg},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{
|
||||
"insecure": {Credentials: insecure.NewBundle()},
|
||||
}),
|
||||
ResourceTypes: map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType},
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
"": {XDSServers: []xdsclient.ServerConfig{serverCfg}},
|
||||
},
|
||||
MetricsReporter: tmr,
|
||||
}
|
||||
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
const listenerName = "test-listener"
|
||||
|
||||
client.WatchResource(listenerType.TypeURL, listenerName, newListenerWatcher())
|
||||
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, "route-config")},
|
||||
SkipValidation: true,
|
||||
}
|
||||
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server: %v", err)
|
||||
}
|
||||
|
||||
if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify that the client substitutes empty authority with '#old' in metrics.
|
||||
tmr.triggerAsyncMetrics()
|
||||
if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{
|
||||
Authority: "#old",
|
||||
ResourceType: "ListenerResource",
|
||||
CacheState: "acked",
|
||||
Count: 1,
|
||||
}); err != nil {
|
||||
t.Fatalf("Failed to observe grpc.xds.authority '#old' metric substitution: %v", err)
|
||||
}
|
||||
}
|
||||
529
internal/xds/clients/xdsclient/test/misc_watchers_test.go
Normal file
529
internal/xds/clients/xdsclient/test/misc_watchers_test.go
Normal file
@@ -0,0 +1,529 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2022 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
"google.golang.org/grpc/internal/xds/clients/grpctransport"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/testutils/fakeserver"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
|
||||
v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
|
||||
)
|
||||
|
||||
// testLDSWatcher is a test watcher that registers two watches corresponding to
|
||||
// the names passed in at creation time on a valid update.
|
||||
type testLDSWatcher struct {
|
||||
client *xdsclient.XDSClient
|
||||
name1, name2 string
|
||||
lw1, lw2 *listenerWatcher
|
||||
cancel1, cancel2 func()
|
||||
updateCh *testutils.Channel
|
||||
}
|
||||
|
||||
func newTestLDSWatcher(client *xdsclient.XDSClient, name1, name2 string) *testLDSWatcher {
|
||||
return &testLDSWatcher{
|
||||
client: client,
|
||||
name1: name1,
|
||||
name2: name2,
|
||||
lw1: newListenerWatcher(),
|
||||
lw2: newListenerWatcher(),
|
||||
updateCh: testutils.NewChannelWithSize(1),
|
||||
}
|
||||
}
|
||||
|
||||
func (lw *testLDSWatcher) ResourceChanged(update xdsclient.ResourceData, onDone func()) {
|
||||
lisData, ok := update.(*listenerResourceData)
|
||||
if !ok {
|
||||
lw.updateCh.Send(listenerUpdateErrTuple{resourceErr: fmt.Errorf("unexpected resource type: %T", update)})
|
||||
onDone()
|
||||
return
|
||||
}
|
||||
lw.updateCh.Send(listenerUpdateErrTuple{update: lisData.Resource})
|
||||
|
||||
lw.cancel1 = lw.client.WatchResource(xdsresource.V3ListenerURL, lw.name1, lw.lw1)
|
||||
lw.cancel2 = lw.client.WatchResource(xdsresource.V3ListenerURL, lw.name2, lw.lw2)
|
||||
onDone()
|
||||
}
|
||||
|
||||
func (lw *testLDSWatcher) AmbientError(err error, onDone func()) {
|
||||
// When used with a go-control-plane management server that continuously
|
||||
// resends resources which are NACKed by the xDS client, using a `Replace()`
|
||||
// here and in OnResourceDoesNotExist() simplifies tests which will have
|
||||
// access to the most recently received error.
|
||||
lw.updateCh.Replace(listenerUpdateErrTuple{ambientErr: err})
|
||||
onDone()
|
||||
}
|
||||
|
||||
func (lw *testLDSWatcher) ResourceError(_ error, onDone func()) {
|
||||
lw.updateCh.Replace(listenerUpdateErrTuple{resourceErr: xdsresource.NewError(xdsresource.ErrorTypeResourceNotFound, "Listener not found in received response")})
|
||||
onDone()
|
||||
}
|
||||
|
||||
func (lw *testLDSWatcher) cancel() {
|
||||
lw.cancel1()
|
||||
lw.cancel2()
|
||||
}
|
||||
|
||||
// TestWatchCallAnotherWatch tests the scenario where a watch is registered for
|
||||
// a resource, and more watches are registered from the first watch's callback.
|
||||
// The test verifies that this scenario does not lead to a deadlock.
|
||||
func (s) TestWatchCallAnotherWatch(t *testing.T) {
|
||||
// Start an xDS management server and set the option to allow it to respond
|
||||
// to requests which only specify a subset of the configured resources.
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
|
||||
|
||||
nodeID := uuid.New().String()
|
||||
authority := makeAuthorityName(t.Name())
|
||||
|
||||
resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
|
||||
si := clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
ResourceTypes: resourceTypes,
|
||||
// Xdstp style resource names used in this test use a slash removed
|
||||
// version of t.Name as their authority, and the empty config
|
||||
// results in the top-level xds server configuration being used for
|
||||
// this authority.
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
authority: {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
}
|
||||
|
||||
// Create an xDS client with the above config.
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// Configure the management server to return two listener resources,
|
||||
// corresponding to the registered watches.
|
||||
ldsNameNewStyle := makeNewStyleLDSName(authority)
|
||||
resources := e2e.UpdateOptions{
|
||||
NodeID: nodeID,
|
||||
Listeners: []*v3listenerpb.Listener{
|
||||
e2e.DefaultClientListener(ldsName, rdsName),
|
||||
e2e.DefaultClientListener(ldsNameNewStyle, rdsNameNewStyle),
|
||||
},
|
||||
SkipValidation: true,
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
if err := mgmtServer.Update(ctx, resources); err != nil {
|
||||
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
||||
}
|
||||
|
||||
// Create a listener watcher that registers two more watches from
|
||||
// the OnUpdate callback:
|
||||
// - one for the same resource name as this watch, which would be
|
||||
// satisfied from xdsClient cache
|
||||
// - the other for a different resource name, which would be
|
||||
// satisfied from the server
|
||||
lw := newTestLDSWatcher(client, ldsName, ldsNameNewStyle)
|
||||
defer lw.cancel()
|
||||
ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, ldsName, lw)
|
||||
defer ldsCancel()
|
||||
|
||||
// Verify the contents of the received update for the all watchers.
|
||||
// Verify the contents of the received update for the all watchers. The two
|
||||
// resources returned differ only in the resource name. Therefore the
|
||||
// expected update is the same for all the watchers.
|
||||
wantUpdate12 := listenerUpdateErrTuple{
|
||||
update: listenerUpdate{
|
||||
RouteConfigName: rdsName,
|
||||
},
|
||||
}
|
||||
// Verify the contents of the received update for the all watchers. The two
|
||||
// resources returned differ only in the resource name. Therefore the
|
||||
// expected update is the same for all the watchers.
|
||||
wantUpdate3 := listenerUpdateErrTuple{
|
||||
update: listenerUpdate{
|
||||
RouteConfigName: rdsNameNewStyle,
|
||||
},
|
||||
}
|
||||
if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate12); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifyListenerUpdate(ctx, lw.lw1.updateCh, wantUpdate12); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifyListenerUpdate(ctx, lw.lw2.updateCh, wantUpdate3); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNodeProtoSentOnlyInFirstRequest verifies that a non-empty node proto gets
|
||||
// sent only on the first discovery request message on the ADS stream.
|
||||
//
|
||||
// It also verifies the same behavior holds after a stream restart.
|
||||
func (s) TestNodeProtoSentOnlyInFirstRequest(t *testing.T) {
|
||||
// Create a restartable listener which can close existing connections.
|
||||
l, err := net.Listen("tcp", "localhost:0")
|
||||
if err != nil {
|
||||
t.Fatalf("Error while listening. Err: %v", err)
|
||||
}
|
||||
lis := testutils.NewRestartableListener(l)
|
||||
|
||||
// Start a fake xDS management server with the above restartable listener.
|
||||
//
|
||||
// We are unable to use the go-control-plane server here, because it caches
|
||||
// the node proto received in the first request message and adds it to
|
||||
// subsequent requests before invoking the OnStreamRequest() callback.
|
||||
// Therefore we cannot verify what is sent by the xDS client.
|
||||
mgmtServer, cleanup, err := fakeserver.StartServer(lis)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to start fake xDS server: %v", err)
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
// Create bootstrap configuration pointing to the above management server.
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
|
||||
si := clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
ResourceTypes: resourceTypes,
|
||||
// Xdstp resource names used in this test do not specify an
|
||||
// authority. These will end up looking up an entry with the
|
||||
// empty key in the authorities map. Having an entry with an
|
||||
// empty key and empty configuration, results in these
|
||||
// resources also using the top-level configuration.
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
"": {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
}
|
||||
|
||||
// Create an xDS client with the above config.
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
const (
|
||||
serviceName = "my-service-client-side-xds"
|
||||
routeConfigName = "route-" + serviceName
|
||||
clusterName = "cluster-" + serviceName
|
||||
serviceName2 = "my-service-client-side-xds-2"
|
||||
)
|
||||
|
||||
// Register a watch for the Listener resource.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
watcher := newListenerWatcher()
|
||||
ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, serviceName, watcher)
|
||||
defer ldsCancel1()
|
||||
|
||||
// Ensure the watch results in a discovery request with an empty node proto.
|
||||
if err := readDiscoveryResponseAndCheckForNonEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Configure a listener resource on the fake xDS server.
|
||||
lisAny, err := anypb.New(e2e.DefaultClientListener(serviceName, routeConfigName))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to marshal listener resource into an Any proto: %v", err)
|
||||
}
|
||||
mgmtServer.XDSResponseChan <- &fakeserver.Response{
|
||||
Resp: &v3discoverypb.DiscoveryResponse{
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
VersionInfo: "1",
|
||||
Resources: []*anypb.Any{lisAny},
|
||||
},
|
||||
}
|
||||
|
||||
// The xDS client is expected to ACK the Listener resource. The discovery
|
||||
// request corresponding to the ACK must contain a nil node proto.
|
||||
if err := readDiscoveryResponseAndCheckForEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Register a watch for another Listener resource.
|
||||
ldscancel2 := client.WatchResource(xdsresource.V3ListenerURL, serviceName2, watcher)
|
||||
defer ldscancel2()
|
||||
|
||||
// Ensure the watch results in a discovery request with an empty node proto.
|
||||
if err := readDiscoveryResponseAndCheckForEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Configure the other listener resource on the fake xDS server.
|
||||
lisAny2, err := anypb.New(e2e.DefaultClientListener(serviceName2, routeConfigName))
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to marshal route configuration resource into an Any proto: %v", err)
|
||||
}
|
||||
|
||||
mgmtServer.XDSResponseChan <- &fakeserver.Response{
|
||||
Resp: &v3discoverypb.DiscoveryResponse{
|
||||
TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
|
||||
VersionInfo: "1",
|
||||
Resources: []*anypb.Any{lisAny2},
|
||||
},
|
||||
}
|
||||
|
||||
// Ensure the discovery request for the ACK contains an empty node proto.
|
||||
if err := readDiscoveryResponseAndCheckForEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Stop the management server and expect the error callback to be invoked.
|
||||
lis.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Timeout when waiting for the connection error to be propagated to the watcher")
|
||||
case <-watcher.ambientErrCh.C:
|
||||
}
|
||||
// Restart the management server.
|
||||
lis.Restart()
|
||||
|
||||
// The xDS client is expected to re-request previously requested resources.
|
||||
// Here, we expect 1 DiscoveryRequest messages with both the listener resources.
|
||||
// The message should contain a non-nil node proto (since its the first
|
||||
// request after restart).
|
||||
//
|
||||
// And since we don't push any responses on the response channel of the fake
|
||||
// server, we do not expect any ACKs here.
|
||||
if err := readDiscoveryResponseAndCheckForNonEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// readDiscoveryResponseAndCheckForEmptyNodeProto reads a discovery request
|
||||
// message out of the provided reqCh. It returns an error if it fails to read a
|
||||
// message before the context deadline expires, or if the read message contains
|
||||
// a non-empty node proto.
|
||||
func readDiscoveryResponseAndCheckForEmptyNodeProto(ctx context.Context, reqCh *testutils.Channel) error {
|
||||
v, err := reqCh.Receive(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Timeout when waiting for a DiscoveryRequest message")
|
||||
}
|
||||
req := v.(*fakeserver.Request).Req.(*v3discoverypb.DiscoveryRequest)
|
||||
if node := req.GetNode(); node != nil {
|
||||
return fmt.Errorf("Node proto received in DiscoveryRequest message is %v, want empty node proto", node)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readDiscoveryResponseAndCheckForNonEmptyNodeProto reads a discovery request
|
||||
// message out of the provided reqCh. It returns an error if it fails to read a
|
||||
// message before the context deadline expires, or if the read message contains
|
||||
// an empty node proto.
|
||||
func readDiscoveryResponseAndCheckForNonEmptyNodeProto(ctx context.Context, reqCh *testutils.Channel) error {
|
||||
v, err := reqCh.Receive(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Timeout when waiting for a DiscoveryRequest message")
|
||||
}
|
||||
req := v.(*fakeserver.Request).Req.(*v3discoverypb.DiscoveryRequest)
|
||||
if node := req.GetNode(); node == nil {
|
||||
return fmt.Errorf("Empty node proto received in DiscoveryRequest message, want non-empty node proto")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Tests that the errors returned by the xDS client when watching a resource
|
||||
// contain the node ID that was used to create the client. This test covers two
|
||||
// scenarios:
|
||||
//
|
||||
// 1. When a watch is registered for an already registered resource type, but
|
||||
// a new watch is registered with a type url which is not present in
|
||||
// provided resource types.
|
||||
// 2. When a watch is registered for a resource name whose authority is not
|
||||
// found in the xDS client config.
|
||||
func (s) TestWatchErrorsContainNodeID(t *testing.T) {
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
|
||||
|
||||
// Create bootstrap configuration pointing to the above management server.
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
|
||||
si := clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
ResourceTypes: resourceTypes,
|
||||
// Xdstp resource names used in this test do not specify an
|
||||
// authority. These will end up looking up an entry with the
|
||||
// empty key in the authorities map. Having an entry with an
|
||||
// empty key and empty configuration, results in these
|
||||
// resources also using the top-level configuration.
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
"": {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
}
|
||||
|
||||
// Create an xDS client with the above config.
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
t.Run("Right_Wrong_ResourceType_Implementations", func(t *testing.T) {
|
||||
const listenerName = "listener-name"
|
||||
watcher := newListenerWatcher()
|
||||
client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher)
|
||||
|
||||
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
||||
defer sCancel()
|
||||
select {
|
||||
case <-sCtx.Done():
|
||||
case <-watcher.updateCh.C:
|
||||
t.Fatal("Unexpected resource update")
|
||||
case <-watcher.resourceErrCh.C:
|
||||
t.Fatal("Unexpected resource error")
|
||||
}
|
||||
|
||||
client.WatchResource("non-existent-type-url", listenerName, watcher)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Timeout when waiting for error callback to be invoked")
|
||||
case u, ok := <-watcher.resourceErrCh.C:
|
||||
if !ok {
|
||||
t.Fatalf("got no update, wanted listener resource error from the management server")
|
||||
}
|
||||
gotErr := u.(listenerUpdateErrTuple).resourceErr
|
||||
if !strings.Contains(gotErr.Error(), nodeID) {
|
||||
t.Fatalf("Unexpected error: %v, want error with node ID: %q", err, nodeID)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Missing_Authority", func(t *testing.T) {
|
||||
const listenerName = "xdstp://nonexistant-authority/envoy.config.listener.v3.Listener/listener-name"
|
||||
watcher := newListenerWatcher()
|
||||
client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Timeout when waiting for error callback to be invoked")
|
||||
case u, ok := <-watcher.resourceErrCh.C:
|
||||
if !ok {
|
||||
t.Fatalf("got no update, wanted listener resource error from the management server")
|
||||
}
|
||||
gotErr := u.(listenerUpdateErrTuple).resourceErr
|
||||
if !strings.Contains(gotErr.Error(), nodeID) {
|
||||
t.Fatalf("Unexpected error: %v, want error with node ID: %q", err, nodeID)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// erroringTransportBuilder is a transport builder which always returns a nil
|
||||
// transport along with an error.
|
||||
type erroringTransportBuilder struct{}
|
||||
|
||||
func (*erroringTransportBuilder) Build(_ clients.ServerIdentifier) (clients.Transport, error) {
|
||||
return nil, fmt.Errorf("failed to create transport")
|
||||
}
|
||||
|
||||
// Tests that the errors returned by the xDS client when watching a resource
|
||||
// contain the node ID when channel creation to the management server fails.
|
||||
func (s) TestWatchErrorsContainNodeID_ChannelCreationFailure(t *testing.T) {
|
||||
mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
|
||||
|
||||
// Create bootstrap configuration pointing to the above management server.
|
||||
nodeID := uuid.New().String()
|
||||
|
||||
resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
|
||||
si := clients.ServerIdentifier{
|
||||
ServerURI: mgmtServer.Address,
|
||||
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
||||
}
|
||||
|
||||
xdsClientConfig := xdsclient.Config{
|
||||
Servers: []xdsclient.ServerConfig{{ServerIdentifier: si}},
|
||||
Node: clients.Node{ID: nodeID},
|
||||
TransportBuilder: &erroringTransportBuilder{},
|
||||
ResourceTypes: resourceTypes,
|
||||
// Xdstp resource names used in this test do not specify an
|
||||
// authority. These will end up looking up an entry with the
|
||||
// empty key in the authorities map. Having an entry with an
|
||||
// empty key and empty configuration, results in these
|
||||
// resources also using the top-level configuration.
|
||||
Authorities: map[string]xdsclient.Authority{
|
||||
"": {XDSServers: []xdsclient.ServerConfig{}},
|
||||
},
|
||||
}
|
||||
|
||||
// Create an xDS client with the above config.
|
||||
client, err := xdsclient.New(xdsClientConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create xDS client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
const listenerName = "listener-name"
|
||||
watcher := newListenerWatcher()
|
||||
client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatal("Timeout when waiting for error callback to be invoked")
|
||||
case u, ok := <-watcher.resourceErrCh.C:
|
||||
if !ok {
|
||||
t.Fatalf("got no update, wanted listener resource error from the management server")
|
||||
}
|
||||
gotErr := u.(listenerUpdateErrTuple).resourceErr
|
||||
if !strings.Contains(gotErr.Error(), nodeID) {
|
||||
t.Fatalf("Unexpected error: %v, want error with node ID: %q", err, nodeID)
|
||||
}
|
||||
}
|
||||
}
|
||||
516
internal/xds/clients/xdsclient/xdsclient.go
Normal file
516
internal/xds/clients/xdsclient/xdsclient.go
Normal file
@@ -0,0 +1,516 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
// Package xdsclient provides an xDS (* Discovery Service) client.
|
||||
//
|
||||
// It allows applications to:
|
||||
// - Create xDS client instances with in-memory configurations.
|
||||
// - Register watches for named resources.
|
||||
// - Receive resources via an ADS (Aggregated Discovery Service) stream.
|
||||
// - Register watches for named resources (e.g. listeners, routes, or
|
||||
// clusters).
|
||||
//
|
||||
// This enables applications to dynamically discover and configure resources
|
||||
// such as listeners, routes, clusters, and endpoints from an xDS management
|
||||
// server.
|
||||
package xdsclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc/internal/grpclog"
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
clientsinternal "google.golang.org/grpc/internal/xds/clients/internal"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/backoff"
|
||||
"google.golang.org/grpc/internal/xds/clients/internal/syncutil"
|
||||
xdsclientinternal "google.golang.org/grpc/internal/xds/clients/xdsclient/internal"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/metrics"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultWatchExpiryTimeout = 15 * time.Second
|
||||
name = "xds-client"
|
||||
)
|
||||
|
||||
var (
|
||||
defaultExponentialBackoff = backoff.DefaultExponential.Backoff
|
||||
)
|
||||
|
||||
func init() {
|
||||
xdsclientinternal.StreamBackoff = defaultExponentialBackoff
|
||||
xdsclientinternal.ResourceWatchStateForTesting = resourceWatchStateForTesting
|
||||
}
|
||||
|
||||
// XDSClient is a client which queries a set of discovery APIs (collectively
|
||||
// termed as xDS) on a remote management server, to discover
|
||||
// various dynamic resources.
|
||||
type XDSClient struct {
|
||||
// The following fields are initialized at creation time and are read-only
|
||||
// after that, and therefore can be accessed without a mutex.
|
||||
done *syncutil.Event // Fired when the client is closed.
|
||||
topLevelAuthority *authority // The top-level authority, used only for old-style names without an authority.
|
||||
authorities map[string]*authority // Map from authority names in config to authority struct.
|
||||
config *Config // Complete xDS client configuration.
|
||||
watchExpiryTimeout time.Duration // Expiry timeout for ADS watch.
|
||||
backoff func(int) time.Duration // Backoff for ADS and LRS stream failures.
|
||||
transportBuilder clients.TransportBuilder // Builder to create transports to xDS server.
|
||||
resourceTypes map[string]ResourceType // Registry of resource types, for parsing incoming ADS responses.
|
||||
serializer *syncutil.CallbackSerializer // Serializer for invoking resource watcher callbacks.
|
||||
serializerClose func() // Function to close the serializer.
|
||||
logger *grpclog.PrefixLogger
|
||||
target string
|
||||
metricsReporter clients.MetricsReporter
|
||||
|
||||
// The XDSClient owns a bunch of channels to individual xDS servers
|
||||
// specified in the xDS client configuration. Authorities acquire references
|
||||
// to these channels based on server configs within the authority config.
|
||||
// The XDSClient maintains a list of interested authorities for each of
|
||||
// these channels, and forwards updates from the channels to each of these
|
||||
// authorities.
|
||||
//
|
||||
// Once all references to a channel are dropped, the channel is closed.
|
||||
channelsMu sync.Mutex
|
||||
xdsActiveChannels map[ServerConfig]*channelState // Map from server config to in-use xdsChannels.
|
||||
|
||||
metricsCleanup func()
|
||||
}
|
||||
|
||||
// New returns a new xDS Client configured with the provided config.
|
||||
func New(config Config) (*XDSClient, error) {
|
||||
switch {
|
||||
case config.ResourceTypes == nil:
|
||||
return nil, errors.New("xdsclient: resource types map is nil")
|
||||
case config.TransportBuilder == nil:
|
||||
return nil, errors.New("xdsclient: transport builder is nil")
|
||||
case config.Authorities == nil && config.Servers == nil:
|
||||
return nil, errors.New("xdsclient: no servers or authorities specified")
|
||||
}
|
||||
if config.WatchExpiryTimeout == 0 {
|
||||
config.WatchExpiryTimeout = defaultWatchExpiryTimeout
|
||||
}
|
||||
client, err := newClient(&config, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Register this client instance as an Async Reporter.
|
||||
if client.metricsReporter != nil {
|
||||
reporter := &xdsClientMetricReporter{c: client}
|
||||
client.metricsCleanup = client.metricsReporter.RegisterAsyncReporter(reporter)
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// newClient returns a new XDSClient with the given config.
|
||||
func newClient(config *Config, target string) (*XDSClient, error) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
c := &XDSClient{
|
||||
target: target,
|
||||
done: syncutil.NewEvent(),
|
||||
authorities: make(map[string]*authority),
|
||||
config: config,
|
||||
watchExpiryTimeout: config.WatchExpiryTimeout,
|
||||
backoff: xdsclientinternal.StreamBackoff,
|
||||
serializer: syncutil.NewCallbackSerializer(ctx),
|
||||
serializerClose: cancel,
|
||||
transportBuilder: config.TransportBuilder,
|
||||
resourceTypes: config.ResourceTypes,
|
||||
xdsActiveChannels: make(map[ServerConfig]*channelState),
|
||||
metricsReporter: config.MetricsReporter,
|
||||
}
|
||||
|
||||
for name, cfg := range config.Authorities {
|
||||
// If server configs are specified in the authorities map, use that.
|
||||
// Else, use the top-level server configs.
|
||||
serverCfg := config.Servers
|
||||
if len(cfg.XDSServers) >= 1 {
|
||||
serverCfg = cfg.XDSServers
|
||||
}
|
||||
c.authorities[name] = newAuthority(authorityBuildOptions{
|
||||
serverConfigs: serverCfg,
|
||||
name: name,
|
||||
serializer: c.serializer,
|
||||
getChannelForADS: c.getChannelForADS,
|
||||
logPrefix: clientPrefix(c),
|
||||
target: target,
|
||||
metricsReporter: c.metricsReporter,
|
||||
})
|
||||
}
|
||||
c.topLevelAuthority = newAuthority(authorityBuildOptions{
|
||||
serverConfigs: config.Servers,
|
||||
name: "",
|
||||
serializer: c.serializer,
|
||||
getChannelForADS: c.getChannelForADS,
|
||||
logPrefix: clientPrefix(c),
|
||||
target: target,
|
||||
metricsReporter: c.metricsReporter,
|
||||
})
|
||||
c.logger = prefixLogger(c)
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Close closes the xDS client and releases all resources.
|
||||
func (c *XDSClient) Close() {
|
||||
if c.done.HasFired() {
|
||||
return
|
||||
}
|
||||
if c.metricsCleanup != nil {
|
||||
c.metricsCleanup()
|
||||
}
|
||||
c.done.Fire()
|
||||
|
||||
c.topLevelAuthority.close()
|
||||
for _, a := range c.authorities {
|
||||
a.close()
|
||||
}
|
||||
|
||||
// Channel close cannot be invoked with the lock held, because it can race
|
||||
// with stream failure happening at the same time. The latter will callback
|
||||
// into the XDSClient and will attempt to grab the lock. This will result
|
||||
// in a deadlock. So instead, we release the lock and wait for all active
|
||||
// channels to be closed.
|
||||
var channelsToClose []*xdsChannel
|
||||
c.channelsMu.Lock()
|
||||
for _, cs := range c.xdsActiveChannels {
|
||||
channelsToClose = append(channelsToClose, cs.channel)
|
||||
}
|
||||
c.xdsActiveChannels = nil
|
||||
c.channelsMu.Unlock()
|
||||
for _, c := range channelsToClose {
|
||||
c.close()
|
||||
}
|
||||
|
||||
c.serializerClose()
|
||||
<-c.serializer.Done()
|
||||
|
||||
c.logger.Infof("Shutdown")
|
||||
}
|
||||
|
||||
// getChannelForADS returns an xdsChannel for the given server configuration.
|
||||
//
|
||||
// If an xdsChannel exists for the given server configuration, it is returned.
|
||||
// Else a new one is created. It also ensures that the calling authority is
|
||||
// added to the set of interested authorities for the returned channel.
|
||||
//
|
||||
// It returns the xdsChannel and a function to release the calling authority's
|
||||
// reference on the channel. The caller must call the cancel function when it is
|
||||
// no longer interested in this channel.
|
||||
//
|
||||
// A non-nil error is returned if an xdsChannel was not created.
|
||||
func (c *XDSClient) getChannelForADS(serverConfig *ServerConfig, callingAuthority *authority) (*xdsChannel, func(), error) {
|
||||
if c.done.HasFired() {
|
||||
return nil, nil, errors.New("xds: the xDS client is closed")
|
||||
}
|
||||
|
||||
initLocked := func(s *channelState) {
|
||||
if c.logger.V(2) {
|
||||
c.logger.Infof("Adding authority %q to the set of interested authorities for channel [%p]", callingAuthority.name, s.channel)
|
||||
}
|
||||
s.interestedAuthorities[callingAuthority] = true
|
||||
}
|
||||
deInitLocked := func(s *channelState) {
|
||||
if c.logger.V(2) {
|
||||
c.logger.Infof("Removing authority %q from the set of interested authorities for channel [%p]", callingAuthority.name, s.channel)
|
||||
}
|
||||
delete(s.interestedAuthorities, callingAuthority)
|
||||
}
|
||||
|
||||
return c.getOrCreateChannel(serverConfig, initLocked, deInitLocked)
|
||||
}
|
||||
|
||||
// getOrCreateChannel returns an xdsChannel for the given server configuration.
|
||||
//
|
||||
// If an active xdsChannel exists for the given server configuration, it is
|
||||
// returned. If an idle xdsChannel exists for the given server configuration, it
|
||||
// is revived from the idle cache and returned. Else a new one is created.
|
||||
//
|
||||
// The initLocked function runs some initialization logic before the channel is
|
||||
// returned. This includes adding the calling authority to the set of interested
|
||||
// authorities for the channel or incrementing the count of the number of LRS
|
||||
// calls on the channel.
|
||||
//
|
||||
// The deInitLocked function runs some cleanup logic when the returned cleanup
|
||||
// function is called. This involves removing the calling authority from the set
|
||||
// of interested authorities for the channel or decrementing the count of the
|
||||
// number of LRS calls on the channel.
|
||||
//
|
||||
// Both initLocked and deInitLocked are called with the c.channelsMu held.
|
||||
//
|
||||
// Returns the xdsChannel and a cleanup function to be invoked when the channel
|
||||
// is no longer required. A non-nil error is returned if an xdsChannel was not
|
||||
// created.
|
||||
func (c *XDSClient) getOrCreateChannel(serverConfig *ServerConfig, initLocked, deInitLocked func(*channelState)) (*xdsChannel, func(), error) {
|
||||
c.channelsMu.Lock()
|
||||
defer c.channelsMu.Unlock()
|
||||
|
||||
if c.logger.V(2) {
|
||||
c.logger.Infof("Received request for a reference to an xdsChannel for server config %q", serverConfig)
|
||||
}
|
||||
|
||||
// Use an existing channel, if one exists for this server config.
|
||||
if st, ok := c.xdsActiveChannels[*serverConfig]; ok {
|
||||
if c.logger.V(2) {
|
||||
c.logger.Infof("Reusing an existing xdsChannel for server config %q", serverConfig)
|
||||
}
|
||||
initLocked(st)
|
||||
return st.channel, c.releaseChannel(serverConfig, st, deInitLocked), nil
|
||||
}
|
||||
|
||||
if c.logger.V(2) {
|
||||
c.logger.Infof("Creating a new xdsChannel for server config %q", serverConfig)
|
||||
}
|
||||
|
||||
// Create a new transport and create a new xdsChannel, and add it to the
|
||||
// map of xdsChannels.
|
||||
tr, err := c.transportBuilder.Build(serverConfig.ServerIdentifier)
|
||||
if err != nil {
|
||||
return nil, func() {}, fmt.Errorf("xds: failed to create transport for server config %v: %v", serverConfig, err)
|
||||
}
|
||||
state := &channelState{
|
||||
parent: c,
|
||||
serverConfig: serverConfig,
|
||||
interestedAuthorities: make(map[*authority]bool),
|
||||
}
|
||||
channel, err := newXDSChannel(xdsChannelOpts{
|
||||
transport: tr,
|
||||
serverConfig: serverConfig,
|
||||
clientConfig: c.config,
|
||||
eventHandler: state,
|
||||
backoff: c.backoff,
|
||||
watchExpiryTimeout: c.watchExpiryTimeout,
|
||||
logPrefix: clientPrefix(c),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, func() {}, fmt.Errorf("xds: failed to create a new channel for server config %v: %v", serverConfig, err)
|
||||
}
|
||||
state.channel = channel
|
||||
c.xdsActiveChannels[*serverConfig] = state
|
||||
initLocked(state)
|
||||
return state.channel, c.releaseChannel(serverConfig, state, deInitLocked), nil
|
||||
}
|
||||
|
||||
// releaseChannel is a function that is called when a reference to an xdsChannel
|
||||
// needs to be released. It handles closing channels with no active references.
|
||||
//
|
||||
// The function takes the following parameters:
|
||||
// - serverConfig: the server configuration for the xdsChannel
|
||||
// - state: the state of the xdsChannel
|
||||
// - deInitLocked: a function that performs any necessary cleanup for the xdsChannel
|
||||
//
|
||||
// The function returns another function that can be called to release the
|
||||
// reference to the xdsChannel. This returned function is idempotent, meaning
|
||||
// it can be called multiple times without any additional effect.
|
||||
func (c *XDSClient) releaseChannel(serverConfig *ServerConfig, state *channelState, deInitLocked func(*channelState)) func() {
|
||||
return sync.OnceFunc(func() {
|
||||
c.channelsMu.Lock()
|
||||
|
||||
if c.logger.V(2) {
|
||||
c.logger.Infof("Received request to release a reference to an xdsChannel for server config %+v", serverConfig)
|
||||
}
|
||||
deInitLocked(state)
|
||||
|
||||
// The channel has active users. Do nothing and return.
|
||||
if len(state.interestedAuthorities) != 0 {
|
||||
if c.logger.V(2) {
|
||||
c.logger.Infof("xdsChannel %p has other active references", state.channel)
|
||||
}
|
||||
c.channelsMu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
delete(c.xdsActiveChannels, *serverConfig)
|
||||
if c.logger.V(2) {
|
||||
c.logger.Infof("Closing xdsChannel [%p] for server config %s", state.channel, serverConfig)
|
||||
}
|
||||
channelToClose := state.channel
|
||||
c.channelsMu.Unlock()
|
||||
|
||||
channelToClose.close()
|
||||
})
|
||||
}
|
||||
|
||||
// DumpResources returns the status and contents of all xDS resources being
|
||||
// watched by the xDS client.
|
||||
func (c *XDSClient) DumpResources() ([]byte, error) {
|
||||
retCfg := c.topLevelAuthority.dumpResources()
|
||||
for _, a := range c.authorities {
|
||||
retCfg = append(retCfg, a.dumpResources()...)
|
||||
}
|
||||
|
||||
nodeProto := clientsinternal.NodeProto(c.config.Node)
|
||||
nodeProto.ClientFeatures = []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper}
|
||||
resp := &v3statuspb.ClientStatusResponse{}
|
||||
resp.Config = append(resp.Config, &v3statuspb.ClientConfig{
|
||||
Node: nodeProto,
|
||||
GenericXdsConfigs: retCfg,
|
||||
})
|
||||
return proto.Marshal(resp)
|
||||
}
|
||||
|
||||
// channelState represents the state of an xDS channel. It tracks the number of
|
||||
// LRS references, the authorities interested in the channel, and the server
|
||||
// configuration used for the channel.
|
||||
//
|
||||
// It receives callbacks for events on the underlying ADS stream and invokes
|
||||
// corresponding callbacks on interested authorities.
|
||||
type channelState struct {
|
||||
parent *XDSClient
|
||||
serverConfig *ServerConfig
|
||||
|
||||
// Access to the following fields should be protected by the parent's
|
||||
// channelsMu.
|
||||
channel *xdsChannel
|
||||
interestedAuthorities map[*authority]bool
|
||||
}
|
||||
|
||||
func (cs *channelState) adsStreamFailure(err error) {
|
||||
if cs.parent.done.HasFired() {
|
||||
return
|
||||
}
|
||||
|
||||
if xdsresource.ErrType(err) != xdsresource.ErrTypeStreamFailedAfterRecv && cs.parent.metricsReporter != nil {
|
||||
cs.parent.metricsReporter.ReportMetric(&metrics.ServerFailure{
|
||||
ServerURI: cs.serverConfig.ServerIdentifier.ServerURI,
|
||||
})
|
||||
}
|
||||
|
||||
cs.parent.channelsMu.Lock()
|
||||
defer cs.parent.channelsMu.Unlock()
|
||||
for authority := range cs.interestedAuthorities {
|
||||
authority.adsStreamFailure(cs.serverConfig, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (cs *channelState) adsResourceUpdate(typ ResourceType, updates map[string]dataAndErrTuple, md xdsresource.UpdateMetadata, onDone func()) {
|
||||
if cs.parent.done.HasFired() {
|
||||
return
|
||||
}
|
||||
|
||||
cs.parent.channelsMu.Lock()
|
||||
defer cs.parent.channelsMu.Unlock()
|
||||
|
||||
if len(cs.interestedAuthorities) == 0 {
|
||||
onDone()
|
||||
return
|
||||
}
|
||||
|
||||
authorityCnt := new(atomic.Int64)
|
||||
authorityCnt.Add(int64(len(cs.interestedAuthorities)))
|
||||
done := func() {
|
||||
if authorityCnt.Add(-1) == 0 {
|
||||
onDone()
|
||||
}
|
||||
}
|
||||
for authority := range cs.interestedAuthorities {
|
||||
authority.adsResourceUpdate(cs.serverConfig, typ, updates, md, done)
|
||||
}
|
||||
}
|
||||
|
||||
func (cs *channelState) adsResourceDoesNotExist(typ ResourceType, resourceName string) {
|
||||
if cs.parent.done.HasFired() {
|
||||
return
|
||||
}
|
||||
|
||||
cs.parent.channelsMu.Lock()
|
||||
defer cs.parent.channelsMu.Unlock()
|
||||
for authority := range cs.interestedAuthorities {
|
||||
authority.adsResourceDoesNotExist(typ, resourceName)
|
||||
}
|
||||
}
|
||||
|
||||
func resourceWatchStateForTesting(c *XDSClient, rType ResourceType, resourceName string) (xdsresource.ResourceWatchState, error) {
|
||||
n := xdsresource.ParseName(resourceName)
|
||||
a := c.getAuthorityForResource(n)
|
||||
if a == nil {
|
||||
return xdsresource.ResourceWatchState{}, fmt.Errorf("unable to find authority for resource name %q", resourceName)
|
||||
}
|
||||
return a.resourceWatchStateForTesting(rType, resourceName)
|
||||
|
||||
}
|
||||
|
||||
// xdsClientMetricReporter is a wrapper around XDSClient used solely for
|
||||
// reporting metrics. We create this separate type to implement the
|
||||
// clients.AsyncReporter interface, preventing its Report method from
|
||||
// becoming part of the public XDSClient API. This is especially important
|
||||
// because the AsyncReporter interface is experimental, and we want to
|
||||
// avoid coupling experimental changes to the stable XDSClient API.
|
||||
type xdsClientMetricReporter struct {
|
||||
c *XDSClient
|
||||
}
|
||||
|
||||
// Report implements clients.AsyncReporter.
|
||||
// This is the entry point invoked by the metrics system during a scrape.
|
||||
func (r *xdsClientMetricReporter) Report(rec clients.AsyncMetricsRecorder) error {
|
||||
r.c.reportConnectedState(rec)
|
||||
r.c.reportResourceStats(rec)
|
||||
return nil
|
||||
}
|
||||
|
||||
// reportConnectedState handles the "grpc.xds_client.connected" metric.
|
||||
func (c *XDSClient) reportConnectedState(rec clients.AsyncMetricsRecorder) {
|
||||
c.channelsMu.Lock()
|
||||
defer c.channelsMu.Unlock()
|
||||
|
||||
for _, cs := range c.xdsActiveChannels {
|
||||
val := int64(0)
|
||||
if cs.channel.ads.isStreamEstablished() {
|
||||
val = 1
|
||||
}
|
||||
|
||||
rec.ReportMetric(&metrics.XDSClientConnected{
|
||||
ServerURI: cs.serverConfig.ServerIdentifier.ServerURI,
|
||||
Value: val,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// reportResourceStats handles the "grpc.xds_client.resources" metric.
|
||||
func (c *XDSClient) reportResourceStats(rec clients.AsyncMetricsRecorder) {
|
||||
reportForAuthority := func(a *authority) {
|
||||
stats := a.resourceStats()
|
||||
for resourceType, stateCounts := range stats {
|
||||
for cacheState, count := range stateCounts {
|
||||
if count > 0 {
|
||||
authorityName := a.name
|
||||
if authorityName == "" {
|
||||
authorityName = "#old"
|
||||
}
|
||||
rec.ReportMetric(&metrics.XDSClientResourceStats{
|
||||
Authority: authorityName,
|
||||
ResourceType: resourceType,
|
||||
CacheState: cacheState,
|
||||
Count: int64(count),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
reportForAuthority(c.topLevelAuthority)
|
||||
for _, a := range c.authorities {
|
||||
reportForAuthority(a)
|
||||
}
|
||||
}
|
||||
128
internal/xds/clients/xdsclient/xdsclient_test.go
Normal file
128
internal/xds/clients/xdsclient/xdsclient_test.go
Normal file
@@ -0,0 +1,128 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
"google.golang.org/grpc/internal/xds/clients/grpctransport"
|
||||
"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
|
||||
)
|
||||
|
||||
func (s) TestXDSClient_New(t *testing.T) {
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
config Config
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "nil resource types",
|
||||
config: Config{
|
||||
Node: clients.Node{ID: "node-id"},
|
||||
},
|
||||
wantErr: "resource types map is nil",
|
||||
},
|
||||
{
|
||||
name: "nil transport builder",
|
||||
config: Config{
|
||||
Node: clients.Node{ID: "node-id"},
|
||||
ResourceTypes: map[string]ResourceType{xdsresource.V3ListenerURL: listenerType},
|
||||
},
|
||||
wantErr: "transport builder is nil",
|
||||
},
|
||||
{
|
||||
name: "no servers or authorities",
|
||||
config: Config{
|
||||
Node: clients.Node{ID: "node-id"},
|
||||
ResourceTypes: map[string]ResourceType{xdsresource.V3ListenerURL: listenerType},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
},
|
||||
wantErr: "no servers or authorities specified",
|
||||
},
|
||||
{
|
||||
name: "success with servers",
|
||||
config: Config{
|
||||
Node: clients.Node{ID: "node-id"},
|
||||
ResourceTypes: map[string]ResourceType{xdsresource.V3ListenerURL: listenerType},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
Servers: []ServerConfig{{ServerIdentifier: clients.ServerIdentifier{ServerURI: "dummy-server"}}},
|
||||
},
|
||||
wantErr: "",
|
||||
},
|
||||
{
|
||||
name: "success with servers and empty nodeID",
|
||||
config: Config{
|
||||
Node: clients.Node{ID: ""},
|
||||
ResourceTypes: map[string]ResourceType{xdsresource.V3ListenerURL: listenerType},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
Servers: []ServerConfig{{ServerIdentifier: clients.ServerIdentifier{ServerURI: "dummy-server"}}},
|
||||
},
|
||||
wantErr: "",
|
||||
},
|
||||
{
|
||||
name: "success with authorities",
|
||||
config: Config{
|
||||
Node: clients.Node{ID: "node-id"},
|
||||
ResourceTypes: map[string]ResourceType{xdsresource.V3ListenerURL: listenerType},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
Authorities: map[string]Authority{"authority-name": {XDSServers: []ServerConfig{{ServerIdentifier: clients.ServerIdentifier{ServerURI: "dummy-server"}}}}},
|
||||
},
|
||||
wantErr: "",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
c, err := New(tt.config)
|
||||
if tt.wantErr == "" {
|
||||
if err != nil {
|
||||
t.Fatalf("New(%+v) failed: %v", tt.config, err)
|
||||
}
|
||||
} else {
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("New(%+v) returned error %v, want error %q", tt.config, err, tt.wantErr)
|
||||
}
|
||||
}
|
||||
if c != nil {
|
||||
c.Close()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s) TestXDSClient_Close(t *testing.T) {
|
||||
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
||||
config := Config{
|
||||
Node: clients.Node{ID: "node-id"},
|
||||
ResourceTypes: map[string]ResourceType{xdsresource.V3ListenerURL: listenerType},
|
||||
TransportBuilder: grpctransport.NewBuilder(configs),
|
||||
Servers: []ServerConfig{{ServerIdentifier: clients.ServerIdentifier{ServerURI: "dummy-server"}}},
|
||||
}
|
||||
c, err := New(config)
|
||||
if err != nil {
|
||||
t.Fatalf("New(%+v) failed: %v", config, err)
|
||||
}
|
||||
c.Close()
|
||||
// Calling close again should not panic.
|
||||
c.Close()
|
||||
}
|
||||
112
internal/xds/clients/xdsclient/xdsconfig.go
Normal file
112
internal/xds/clients/xdsclient/xdsconfig.go
Normal file
@@ -0,0 +1,112 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package xdsclient
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc/internal/xds/clients"
|
||||
)
|
||||
|
||||
// ServerFeature indicates the features that will be supported by an xDS server.
|
||||
type ServerFeature uint64
|
||||
|
||||
const (
|
||||
// ServerFeatureIgnoreResourceDeletion indicates that the server supports a
|
||||
// feature where the xDS client can ignore resource deletions from this server,
|
||||
// as described in gRFC A53.
|
||||
ServerFeatureIgnoreResourceDeletion ServerFeature = 1 << iota
|
||||
// ServerFeatureTrustedXDSServer returns true if this server is trusted,
|
||||
// and gRPC should accept security-config-affecting fields from the server
|
||||
// as described in gRFC A81.
|
||||
ServerFeatureTrustedXDSServer
|
||||
)
|
||||
|
||||
// Config is used to configure an xDS client. After one has been passed to the
|
||||
// xDS client's New function, no part of it may be modified. A Config may be
|
||||
// reused; the xdsclient package will also not modify it.
|
||||
type Config struct {
|
||||
// Servers specifies a list of xDS management servers to connect to. The
|
||||
// order of the servers in this list reflects the order of preference of
|
||||
// the data returned by those servers. The xDS client uses the first
|
||||
// available server from the list.
|
||||
//
|
||||
// See gRFC A71 for more details on fallback behavior when the primary
|
||||
// xDS server is unavailable.
|
||||
//
|
||||
// gRFC A71: https://github.com/grpc/proposal/blob/master/A71-xds-fallback.md
|
||||
Servers []ServerConfig
|
||||
|
||||
// Authorities defines the configuration for each xDS authority. Federated resources
|
||||
// will be fetched from the servers specified by the corresponding Authority.
|
||||
Authorities map[string]Authority
|
||||
|
||||
// Node is the identity of the xDS client connecting to the xDS
|
||||
// management server.
|
||||
Node clients.Node
|
||||
|
||||
// TransportBuilder is used to create connections to xDS management servers.
|
||||
TransportBuilder clients.TransportBuilder
|
||||
|
||||
// ResourceTypes is a map from resource type URLs to resource type
|
||||
// implementations. Each resource type URL uniquely identifies a specific
|
||||
// kind of xDS resource, and the corresponding resource type implementation
|
||||
// provides logic for parsing, validating, and processing resources of that
|
||||
// type.
|
||||
//
|
||||
// For example: "type.googleapis.com/envoy.config.listener.v3.Listener"
|
||||
ResourceTypes map[string]ResourceType
|
||||
|
||||
// MetricsReporter is used to report registered metrics. If unset, no
|
||||
// metrics will be reported.
|
||||
MetricsReporter clients.MetricsReporter
|
||||
|
||||
// WatchExpiryTimeout is the duration after which a resource watch expires
|
||||
// if the requested resource is not received from the management server.
|
||||
// Most users will not need to set this. If zero, a default value of 15
|
||||
// seconds is used as specified here:
|
||||
// envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#knowing-when-a-requested-resource-does-not-exist
|
||||
WatchExpiryTimeout time.Duration
|
||||
}
|
||||
|
||||
// ServerConfig contains configuration for an xDS management server.
|
||||
type ServerConfig struct {
|
||||
ServerIdentifier clients.ServerIdentifier
|
||||
ServerFeature ServerFeature // ServerFeature stores a bitmap of supported features.
|
||||
}
|
||||
|
||||
// Authority contains configuration for an xDS control plane authority.
|
||||
//
|
||||
// See: https://www.envoyproxy.io/docs/envoy/latest/xds/core/v3/resource_locator.proto#xds-core-v3-resourcelocator
|
||||
type Authority struct {
|
||||
// XDSServers contains the list of server configurations for this authority.
|
||||
//
|
||||
// See Config.Servers for more details.
|
||||
XDSServers []ServerConfig
|
||||
}
|
||||
|
||||
func isServerConfigEqual(a, b *ServerConfig) bool {
|
||||
return a.ServerIdentifier == b.ServerIdentifier && a.ServerFeature == b.ServerFeature
|
||||
}
|
||||
|
||||
// SupportsServerFeature returns true if the server configuration indicates that
|
||||
// the server supports the given feature.
|
||||
func (s *ServerConfig) SupportsServerFeature(feature ServerFeature) bool {
|
||||
return s.ServerFeature&feature != 0
|
||||
}
|
||||
Reference in New Issue
Block a user